You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2023/03/27 08:00:45 UTC
[linkis] branch dev-1.3.2 updated: update jdbc connect logic (#4412)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.2
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.3.2 by this push:
new 7005c01d7 update jdbc connect logic (#4412)
7005c01d7 is described below
commit 7005c01d7f7bca78322447f4f2f32b8398645687
Author: aiceflower <ki...@gmail.com>
AuthorDate: Mon Mar 27 16:00:37 2023 +0800
update jdbc connect logic (#4412)
* update jdbc connect logic
---
.../apache/linkis/common/utils/SecurityUtils.java | 194 ++++++++----
.../linkis/common/utils/SecurityUtilsTest.java | 338 +++++++++++++++------
.../engineplugin/jdbc/ConnectionManager.java | 13 +-
.../engineplugin/jdbc/utils/JdbcParamUtils.java | 39 ---
.../jdbc/utils/JdbcParamUtilsTest.java | 124 --------
.../query/service/mysql/SqlConnection.java | 36 +--
6 files changed, 408 insertions(+), 336 deletions(-)
diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java
index 2e77a14c0..c145f4728 100644
--- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java
+++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java
@@ -25,10 +25,9 @@ import org.apache.commons.lang3.StringUtils;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@@ -46,8 +45,12 @@ public abstract class SecurityUtils {
private static final String QUESTION_MARK = "?";
+ private static final String REGEX_QUESTION_MARK = "\\?";
+
+ private static final int JDBC_URL_ITEM_COUNT = 2;
+
/** allowLoadLocalInfile,allowLoadLocalInfiled,# */
- public static final CommonVars<String> MYSQL_SENSITIVE_PARAMS =
+ private static final CommonVars<String> MYSQL_SENSITIVE_PARAMS =
CommonVars$.MODULE$.apply(
"linkis.mysql.sensitive.params",
"allowLoadLocalInfile,autoDeserialize,allowLocalInfile,allowUrlInLocalInfile,#");
@@ -55,16 +58,113 @@ public abstract class SecurityUtils {
/**
* "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"
*/
- public static final CommonVars<String> MYSQL_FORCE_PARAMS =
+ private static final CommonVars<String> MYSQL_FORCE_PARAMS =
CommonVars$.MODULE$.apply(
"linkis.mysql.force.params",
"allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false");
- public static final CommonVars<String> MYSQL_STRONG_SECURITY_ENABLE =
+ private static final CommonVars<String> MYSQL_STRONG_SECURITY_ENABLE =
CommonVars$.MODULE$.apply("linkis.mysql.strong.security.enable", "false");
+ private static final CommonVars<String> MYSQL_CONNECT_URL =
+ CommonVars.apply("linkis.security.mysql.url.template", "jdbc:mysql://%s:%s/%s");
+
+ private static final String JDBC_MATCH_REGEX =
+ "(?i)jdbc:(?i)(mysql)://([a-zA-Z0-9]+\\.)*[a-zA-Z0-9]+(:[0-9]+)?(/[a-zA-Z0-9_-]*[\\.\\-]?)?";
+
+ private static final String JDBC_MYSQL_PROTOCOL = "jdbc:mysql";
+
+ /**
+ * check mysql connection params
+ *
+ * @param host
+ * @param port
+ * @param username
+ * @param password
+ * @param database
+ * @param extraParams
+ */
+ public static void checkJdbcConnParams(
+ String host,
+ Integer port,
+ String username,
+ String password,
+ String database,
+ Map<String, Object> extraParams) {
+ // 1. Check blank params
+ if (StringUtils.isAnyBlank(host, username, password)) {
+ logger.info(
+ "Invalid mysql connection params: host: {}, username: {}, password: {}, database: {}",
+ host,
+ username,
+ password,
+ database);
+ throw new LinkisSecurityException(35000, "Invalid mysql connection params.");
+ }
+
+ // 2. Check url format
+ String url = String.format(MYSQL_CONNECT_URL.getValue(), host.trim(), port, database.trim());
+ checkUrl(url);
+
+ // 3. Check params. Mainly vulnerability parameters. Note the url encoding
+ checkParams(extraParams);
+ }
+
+ /** @param url */
+ public static void checkJdbcConnUrl(String url) {
+ logger.info("jdbc url: {}", url);
+ if (StringUtils.isBlank(url)) {
+ throw new LinkisSecurityException(35000, "Invalid jdbc connection url.");
+ }
+
+ // temporarily only check mysql jdbc url.
+ if (!url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) {
+ return;
+ }
+
+ String[] urlItems = url.split(REGEX_QUESTION_MARK);
+ if (urlItems.length > JDBC_URL_ITEM_COUNT) {
+ throw new LinkisSecurityException(35000, "Invalid jdbc connection url.");
+ }
+
+ // check url
+ checkUrl(urlItems[0]);
+
+ // check params
+ if (urlItems.length == JDBC_URL_ITEM_COUNT) {
+ Map<String, Object> params = parseMysqlUrlParamsToMap(urlItems[1]);
+ checkParams(params);
+ }
+ }
+
+ /**
+ * call after checkJdbcConnUrl
+ *
+ * @param url
+ * @return
+ */
+ public static String getJdbcUrl(String url) {
+ // preventing NPE
+ if (StringUtils.isBlank(url)) {
+ return url;
+ }
+ // temporarily deal with only mysql jdbc url.
+ if (!url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) {
+ return url;
+ }
+ String[] items = url.split(REGEX_QUESTION_MARK);
+ String result = items[0];
+ if (items.length == JDBC_URL_ITEM_COUNT) {
+ Map<String, Object> params = parseMysqlUrlParamsToMap(items[1]);
+ appendMysqlForceParams(params);
+ String paramUrl = parseParamsMapToMysqlParamUrl(params);
+ result += QUESTION_MARK + paramUrl;
+ }
+ return result;
+ }
+
/**
- * mysql url append force params
+ * append force params, Should be called after the checkJdbcConnParams method
*
* @param url
* @return
@@ -73,6 +173,9 @@ public abstract class SecurityUtils {
if (StringUtils.isBlank(url)) {
return "";
}
+ if (!Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) {
+ return url;
+ }
String extraParamString = MYSQL_FORCE_PARAMS.getValue();
@@ -86,36 +189,41 @@ public abstract class SecurityUtils {
return url;
}
+ /**
+ * append force params, Should be called after the checkJdbcConnParams method
+ *
+ * @param extraParams
+ */
public static void appendMysqlForceParams(Map<String, Object> extraParams) {
- extraParams.putAll(parseMysqlUrlParamsToMap(MYSQL_FORCE_PARAMS.getValue()));
+ if (Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) {
+ extraParams.putAll(parseMysqlUrlParamsToMap(MYSQL_FORCE_PARAMS.getValue()));
+ }
}
- public static String checkJdbcSecurity(String url) {
- logger.info("checkJdbcSecurity origin url: {}", url);
- if (StringUtils.isBlank(url)) {
- throw new LinkisSecurityException(35000, "Invalid mysql connection cul, url is empty");
- }
- // deal with url encode
- try {
- url = URLDecoder.decode(url, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new LinkisSecurityException(35000, "mysql connection cul decode error: " + e);
+ public static String parseParamsMapToMysqlParamUrl(Map<String, Object> params) {
+ if (params == null || params.isEmpty()) {
+ return "";
}
- if (url.endsWith(QUESTION_MARK) || !url.contains(QUESTION_MARK)) {
- logger.info("checkJdbcSecurity target url: {}", url);
- return url;
+ return params.entrySet().stream()
+ .map(e -> String.join(EQUAL_SIGN, e.getKey(), String.valueOf(e.getValue())))
+ .collect(Collectors.joining(AND_SYMBOL));
+ }
+
+ /**
+ * check url, format: jdbc:mysql://host:port/dbname
+ *
+ * @param url
+ */
+ public static void checkUrl(String url) {
+ if (url != null && !url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) {
+ return;
}
- String[] items = url.split("\\?");
- if (items.length != 2) {
- logger.warn("Invalid url: {}", url);
- throw new LinkisSecurityException(35000, "Invalid mysql connection cul: " + url);
+ Pattern regex = Pattern.compile(JDBC_MATCH_REGEX);
+ Matcher matcher = regex.matcher(url);
+ if (!matcher.matches()) {
+ logger.info("Invalid mysql connection url: {}", url);
+ throw new LinkisSecurityException(35000, "Invalid mysql connection url.");
}
- Map<String, Object> params = parseMysqlUrlParamsToMap(items[1]);
- Map<String, Object> securityMap = checkJdbcSecurity(params);
- String paramUrl = parseParamsMapToMysqlParamUrl(securityMap);
- url = items[0] + QUESTION_MARK + paramUrl;
- logger.info("checkJdbcSecurity target url: {}", url);
- return url;
}
/**
@@ -123,15 +231,9 @@ public abstract class SecurityUtils {
*
* @param paramsMap
*/
- public static Map<String, Object> checkJdbcSecurity(Map<String, Object> paramsMap) {
- if (paramsMap == null) {
- return new HashMap<>();
- }
-
- // mysql url strong security
- if (Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) {
- paramsMap.clear();
- return paramsMap;
+ private static void checkParams(Map<String, Object> paramsMap) {
+ if (paramsMap == null || paramsMap.isEmpty()) {
+ return;
}
// deal with url encode
@@ -163,16 +265,6 @@ public abstract class SecurityUtils {
"Invalid mysql connection parameters: " + parseParamsMapToMysqlParamUrl(paramsMap));
}
}
- return paramsMap;
- }
-
- public static String parseParamsMapToMysqlParamUrl(Map<String, Object> forceParams) {
- if (forceParams == null) {
- return "";
- }
- return forceParams.entrySet().stream()
- .map(e -> String.join(EQUAL_SIGN, e.getKey(), String.valueOf(e.getValue())))
- .collect(Collectors.joining(AND_SYMBOL));
}
private static Map<String, Object> parseMysqlUrlParamsToMap(String paramsUrl) {
diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java
index 4fdca7b82..ed06f9da8 100644
--- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java
+++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java
@@ -17,166 +17,326 @@
package org.apache.linkis.common.utils;
+import org.apache.linkis.common.conf.BDPConfiguration;
import org.apache.linkis.common.exception.LinkisSecurityException;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
/** SecurityUtils Tester */
public class SecurityUtilsTest {
+ @BeforeAll
+ public static void init() {
+ BDPConfiguration.set("linkis.mysql.strong.security.enable", "true");
+ }
+
@Test
- public void testAppendMysqlForceParamsUrl() throws Exception {
- // allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false
+ public void testCheckUrl() {
+ // true
String url = "jdbc:mysql://127.0.0.1:10000/db_name";
- String newUrl = SecurityUtils.appendMysqlForceParams(url);
- Assertions.assertEquals(
- url
- + "?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
- newUrl);
-
- url = "jdbc:mysql://127.0.0.1:10000/db_name?";
- newUrl = SecurityUtils.appendMysqlForceParams(url);
- Assertions.assertEquals(
- url
- + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
- newUrl);
-
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1";
- newUrl = SecurityUtils.appendMysqlForceParams(url);
- Assertions.assertEquals(
- url
- + "&"
- + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
- newUrl);
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkUrl(url);
+ });
+ // false
+ String url1 = "jdbc:mysql://127.0.0.1:10000/db_name?";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkUrl(url1);
+ });
+ // false
+ String url11 = "jdbc:mysql://127.0.0.1:10000/db_name?abc";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkUrl(url11);
+ });
+ // true
+ String url2 = "jdbc:mysql://127.0.0.1:10000/";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkUrl(url2);
+ });
+ // true
+ String url3 = "jdbc:mysql://127.0.0.1:10000";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkUrl(url3);
+ });
+ // true
+ String url4 = "JDBC:mysql://127.0.0.1:10000/db_name";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkUrl(url4);
+ });
+ // true
+ String url5 = "JDBC:H2://127.0.0.1:10000/db_name";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkUrl(url5);
+ });
}
@Test
- public void testAppendMysqlForceParamsExtraParams() throws Exception {
- Map<String, Object> extraParams = new HashMap<>();
- extraParams.put("testKey", "testValue");
- SecurityUtils.appendMysqlForceParams(extraParams);
- Assertions.assertEquals("false", extraParams.get("allowLoadLocalInfile"));
- Assertions.assertEquals("false", extraParams.get("autoDeserialize"));
- Assertions.assertEquals("false", extraParams.get("allowLocalInfile"));
- Assertions.assertEquals("false", extraParams.get("allowUrlInLocalInfile"));
- Assertions.assertEquals("testValue", extraParams.get("testKey"));
- Assertions.assertEquals(null, extraParams.get("otherKey"));
+ public void testGetUrl() {
+ BDPConfiguration.set("linkis.mysql.strong.security.enable", "true");
+ String baseUrl = "jdbc:mysql://127.0.0.1:10000/db_name";
+ String securityStr =
+ "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false";
+ String url1 = "jdbc:mysql://127.0.0.1:10000/db_name";
+ Assertions.assertEquals(baseUrl, SecurityUtils.getJdbcUrl(url1));
+ String url11 = "jdbc:mysql://127.0.0.1:10000/db_name?";
+ Assertions.assertEquals(baseUrl, SecurityUtils.getJdbcUrl(url11));
+ String url2 = "jdbc:mysql://127.0.0.1:10000/db_name?k1=v1&";
+ Assertions.assertEquals(baseUrl + "?k1=v1&" + securityStr, SecurityUtils.getJdbcUrl(url2));
+ String url3 = "jdbc:mysql://127.0.0.1:10000/db_name?k1=v1&k2";
+ Assertions.assertEquals(baseUrl + "?k1=v1&" + securityStr, SecurityUtils.getJdbcUrl(url3));
}
@Test
- public void testCheckJdbcSecurityUrl() throws Exception {
- String url = "jdbc:mysql://127.0.0.1:10000/db_name";
- String newUrl = SecurityUtils.checkJdbcSecurity(url);
- Assertions.assertEquals(url, newUrl);
+ public void testCheckJdbcConnParams() {
+ String host = "127.0.0.1";
+ Integer port = 3306;
+ String username = "test";
+ String password = "test";
+ String database = "tdb";
+ Map<String, Object> extraParams = new HashMap<>();
+ extraParams.put("k1", "v1");
- url = "jdbc:mysql://127.0.0.1:10000/db_name?";
- newUrl = SecurityUtils.checkJdbcSecurity(url);
- Assertions.assertEquals(url, newUrl);
+ // match ip
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+ String host1 = "localhost";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host1, port, username, password, database, extraParams);
+ });
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1";
- newUrl = SecurityUtils.checkJdbcSecurity(url);
- Assertions.assertEquals(url, newUrl);
+ // match domain
+ String host2 = "www.apache.com";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host2, port, username, password, database, extraParams);
+ });
- // key is not security
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true";
- AtomicReference<String> atomUrl = new AtomicReference<>(url);
+ // error host
+ String host3 = "localhost:3306";
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(atomUrl.get());
+ SecurityUtils.checkJdbcConnParams(host3, port, username, password, database, extraParams);
});
- // url encode
- url = "jdbc:mysql://127.0.0.1:10000/db_name?allowLocalInfil%65=true";
- atomUrl.set(url);
+ String host4 = "localhost:3306/test";
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(atomUrl.get());
+ SecurityUtils.checkJdbcConnParams(host4, port, username, password, database, extraParams);
});
- // value is not security
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile";
- atomUrl.set(url);
+ String host5 = "localhost/test";
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(atomUrl.get());
+ SecurityUtils.checkJdbcConnParams(host5, port, username, password, database, extraParams);
});
- // contains #
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2";
- atomUrl.set(url);
+ // error port
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(atomUrl.get());
+ SecurityUtils.checkJdbcConnParams(host, null, username, password, database, extraParams);
});
- }
- @Test
- public void testCheckJdbcSecurityParamsMap() throws Exception {
- Map<String, Object> paramsMap = new HashMap<>();
- paramsMap.put("p1", "v1");
- Map<String, Object> newMap = SecurityUtils.checkJdbcSecurity(paramsMap);
- Assertions.assertEquals("v1", newMap.get("p1"));
+ // error username
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, " ", password, database, extraParams);
+ });
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, null, password, database, extraParams);
+ });
- // key not security
- paramsMap.put("allowLocalInfil%67", "true");
- SecurityUtils.checkJdbcSecurity(paramsMap);
- Assertions.assertEquals("true", newMap.get("allowLocalInfilg"));
+ // error password
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, " ", database, extraParams);
+ });
- // key not security
- paramsMap.put("allowLocalInfile", "false");
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(paramsMap);
+ SecurityUtils.checkJdbcConnParams(host, port, username, null, database, extraParams);
});
- // value not security
- paramsMap.clear();
- paramsMap.put("p1", "allowLocalInfile");
+ // check database, The database name can be empty
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, " ", extraParams);
+ });
+
+ String database1 = "test?k1=v1";
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(paramsMap);
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database1, extraParams);
});
- // value not security
- paramsMap.clear();
- paramsMap.put("p1", "allowLocalInfil%65");
+ // error param
+ extraParams.put("allowLoadLocalInfile", "true");
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(paramsMap);
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
});
- // contains #
- paramsMap.clear();
- paramsMap.put("p1#", "v1");
+ extraParams.clear();
+ extraParams.put("autoDeserialize", "true");
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(paramsMap);
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
});
- paramsMap.clear();
- paramsMap.put("p1", "v1#");
+ extraParams.clear();
+ extraParams.put("allowLocalInfile", "true");
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+
+ extraParams.clear();
+ extraParams.put("allowUrlInLocalInfile", "false");
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+
+ extraParams.clear();
+ extraParams.put("allowLocalInfil%65", "true");
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+
+ extraParams.clear();
+ extraParams.put("#", "true");
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+
+ extraParams.clear();
+ extraParams.put("test", "#");
Assertions.assertThrows(
LinkisSecurityException.class,
() -> {
- SecurityUtils.checkJdbcSecurity(paramsMap);
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
+ });
+ }
+
+ @Test
+ public void testCheckJdbcConnUrl() {
+ // true
+ String url = "jdbc:mysql://127.0.0.1:10000/db_name";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url);
+ });
+ // true
+ String url1 = "jdbc:mysql://127.0.0.1:10000/db_name?";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url1);
+ });
+ // true
+ String url11 = "jdbc:mysql://127.0.0.1/db_name?";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url11);
+ });
+ // true
+ String url2 = "JDBC:mysql://127.0.0.1:10000/db_name?";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url2);
+ });
+ // true
+ String url21 = "JDBC:h2://127.0.0.1:10000/db_name?";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url21);
});
+ // true
+ String url3 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1";
+ Assertions.assertDoesNotThrow(
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url3);
+ });
+ // false url error
+ String url33 =
+ "jdbc:mysql://127.0.0.1:10000:/db_name?jdbc:mysql://127.0.0.1:10000?allowLocalInfile=true";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url33);
+ });
+ // false key is not security
+ String url4 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url4);
+ });
+
+ // false value is not security
+ String url5 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url5);
+ });
+
+ // false contains #
+ String url6 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2";
+ Assertions.assertThrows(
+ LinkisSecurityException.class,
+ () -> {
+ SecurityUtils.checkJdbcConnUrl(url6);
+ });
+ }
+
+ @Test
+ public void testAppendMysqlForceParamsExtraParams() {
+ Map<String, Object> extraParams = new HashMap<>();
+ extraParams.put("testKey", "testValue");
+ SecurityUtils.appendMysqlForceParams(extraParams);
+ Assertions.assertEquals("false", extraParams.get("allowLoadLocalInfile"));
+ Assertions.assertEquals("false", extraParams.get("autoDeserialize"));
+ Assertions.assertEquals("false", extraParams.get("allowLocalInfile"));
+ Assertions.assertEquals("false", extraParams.get("allowUrlInLocalInfile"));
+ Assertions.assertEquals("testValue", extraParams.get("testKey"));
+ Assertions.assertEquals(null, extraParams.get("otherKey"));
}
@Test
- public void testMapToString() throws Exception {
+ public void testMapToString() {
Map<String, Object> paramsMap = new HashMap<>();
paramsMap.put("p1", "v1");
String str = SecurityUtils.parseParamsMapToMysqlParamUrl(paramsMap);
diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java
index 73fdf1ada..b9cd47945 100644
--- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java
+++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java
@@ -17,6 +17,7 @@
package org.apache.linkis.manager.engineplugin.jdbc;
+import org.apache.linkis.common.utils.SecurityUtils;
import org.apache.linkis.hadoop.common.utils.KerberosUtils;
import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant;
import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException;
@@ -296,12 +297,12 @@ public class ConnectionManager {
private String getJdbcUrl(Map<String, String> properties) throws SQLException {
String url = properties.get(JDBCEngineConnConstant.JDBC_URL);
if (StringUtils.isBlank(url)) {
- throw new SQLException(JDBCEngineConnConstant.JDBC_URL + " is not empty.");
+ throw new SQLException(JDBCEngineConnConstant.JDBC_URL + " cannot be empty.");
}
url = JdbcParamUtils.clearJdbcUrl(url);
- url = JdbcParamUtils.filterJdbcUrl(url);
- JdbcParamUtils.validateJdbcUrl(url);
- return url.trim();
+ SecurityUtils.checkJdbcConnUrl(url);
+ url = SecurityUtils.getJdbcUrl(url);
+ return url;
}
private String appendProxyUserToJDBCUrl(
@@ -329,7 +330,9 @@ public class ConnectionManager {
private JdbcAuthType getJdbcAuthType(Map<String, String> properties) {
String authType =
properties.getOrDefault(JDBCEngineConnConstant.JDBC_AUTH_TYPE, USERNAME.getAuthType());
- if (authType == null || authType.trim().length() == 0) return of(USERNAME.getAuthType());
+ if (authType == null || authType.trim().length() == 0) {
+ return of(USERNAME.getAuthType());
+ }
return of(authType.trim().toUpperCase());
}
diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java
index d4ddee737..850513637 100644
--- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java
+++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java
@@ -19,7 +19,6 @@ package org.apache.linkis.manager.engineplugin.jdbc.utils;
import org.apache.linkis.common.conf.CommonVars;
import org.apache.linkis.common.conf.CommonVars$;
-import org.apache.linkis.common.utils.SecurityUtils;
import org.apache.linkis.manager.engineplugin.jdbc.JDBCPropertiesParser;
import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant;
import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException;
@@ -35,12 +34,6 @@ import static org.apache.linkis.manager.engineplugin.jdbc.errorcode.JDBCErrorCod
public class JdbcParamUtils {
private static final Logger LOG = LoggerFactory.getLogger(JdbcParamUtils.class);
- private static final String JDBC_MATCH_REGEX = "jdbc:\\w+://\\S+:[0-9]{2,6}(/\\S*)?(\\?\\S*)?";
- private static final String JDBC_H2_PROTOCOL = "jdbc:h2";
-
- private static final String JDBC_MYSQL_PROTOCOL = "jdbc:mysql";
-
- private static final String SENSITIVE_PARAM = "autoDeserialize=true";
private static final String AUTO_DESERIALIZE = "autoDeserialize";
private static final String APPEND_PARAMS =
@@ -49,12 +42,8 @@ public class JdbcParamUtils {
public static final CommonVars<String> MYSQL_STRONG_SECURITY_ENABLE =
CommonVars$.MODULE$.apply("linkis.mysql.strong.security.enable", "false");
- private static final char AND_SYMBOL = '&';
-
private static final String QUOTATION_MARKS = "\"";
- private static final char QUESTION_MARK = '?';
-
public static String clearJdbcUrl(String url) {
if (url.startsWith(QUOTATION_MARKS) && url.endsWith(QUOTATION_MARKS)) {
url = url.trim();
@@ -63,34 +52,6 @@ public class JdbcParamUtils {
return url;
}
- public static void validateJdbcUrl(String url) {
- if (!url.matches(JDBC_MATCH_REGEX) && !url.startsWith(JDBC_H2_PROTOCOL)) {
- throw new IllegalArgumentException("JDBC url format error!" + url);
- }
- }
-
- public static String filterJdbcUrl(String url) {
-
- LOG.info("the filter source url is: {}", url);
-
- if (StringUtils.isBlank(url)) {
- return url;
- }
- // temporarily filter only mysql jdbc url. & Handles cases that start with JDBC
- if (!url.toLowerCase().contains(JDBC_MYSQL_PROTOCOL)) {
- return url;
- }
-
- // security check
- url = SecurityUtils.checkJdbcSecurity(url);
-
- // append force params
- url = SecurityUtils.appendMysqlForceParams(url);
-
- LOG.info("The filtered jdbc url is: {}", url);
- return url;
- }
-
public static String getJdbcUsername(Map<String, String> properties)
throws JDBCParamsIllegalException {
String username =
diff --git a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java
index 701f94eff..78984e01e 100644
--- a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java
+++ b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java
@@ -17,141 +17,17 @@
package org.apache.linkis.manager.engineplugin.jdbc.utils;
-import org.apache.linkis.common.exception.LinkisSecurityException;
import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant;
import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
public class JdbcParamUtilsTest {
- @Test
- @DisplayName("testFilterJdbcUrl")
- public void testFilterJdbcUrl() {
- String securityParam =
- "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false";
- String url = "jdbc:mysql://127.0.0.1:10000/db_name";
- String newUrl = JdbcParamUtils.filterJdbcUrl(url);
- Assertions.assertEquals(url + "?" + securityParam, newUrl);
-
- // not mysql url
- url = "h2:mysql";
- newUrl = JdbcParamUtils.filterJdbcUrl(url);
- Assertions.assertEquals(url, newUrl);
-
- // start with JDBC
- url = "JDBC:mysql://127.0.0.1:10000/db_name?";
- newUrl = JdbcParamUtils.filterJdbcUrl(url);
- Assertions.assertEquals(url + securityParam, newUrl);
-
- url = "jdbc:mysql://127.0.0.1:10000/db_name?";
- newUrl = JdbcParamUtils.filterJdbcUrl(url);
- Assertions.assertEquals(url + securityParam, newUrl);
-
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1";
- newUrl = JdbcParamUtils.filterJdbcUrl(url);
- Assertions.assertEquals(url + "&" + securityParam, newUrl);
-
- // key is not security
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true";
- AtomicReference<String> atomUrl = new AtomicReference<>(url);
- Assertions.assertThrows(
- LinkisSecurityException.class,
- () -> {
- JdbcParamUtils.filterJdbcUrl(atomUrl.get());
- });
-
- // value is not security
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile";
- atomUrl.set(url);
- Assertions.assertThrows(
- LinkisSecurityException.class,
- () -> {
- JdbcParamUtils.filterJdbcUrl(atomUrl.get());
- });
-
- // contains #
- url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2";
- atomUrl.set(url);
- Assertions.assertThrows(
- LinkisSecurityException.class,
- () -> {
- JdbcParamUtils.filterJdbcUrl(atomUrl.get());
- });
- }
-
- @Test
- @DisplayName("testValidateJdbcUrl")
- public void testValidateJdbcUrl() {
- AtomicReference<String> ar = new AtomicReference<>();
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/abc?p1=v1&p2=v2");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/?p1=v1&p2=v2");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000?p1=v1&p2=v2");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/abc?");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/abc");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // true
- ar.set("jdbc:mysql://127.0.0.1:10000/?v1=v2");
- Assertions.assertDoesNotThrow(
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // false
- ar.set("jdbc:mysql://127.0.0.1:10000000/?v1=v2");
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- // false
- ar.set("jdbc:mysql://127.0.0.1:10000ab/?v1=v2");
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () -> {
- JdbcParamUtils.validateJdbcUrl(ar.get());
- });
- }
@Test
@DisplayName("testGetJdbcUsername")
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java
index b5a3a54fa..bab34059e 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java
@@ -48,9 +48,6 @@ public class SqlConnection implements Closeable {
private static final CommonVars<Integer> SQL_SOCKET_TIMEOUT =
CommonVars.apply("wds.linkis.server.mdm.service.sql.socket.timeout", 6000);
- private static final CommonVars<Boolean> MYSQL_STRONG_SECURITY_ENABLE =
- CommonVars.apply("linkis.mysql.strong.security.enable", false);
-
private Connection conn;
private ConnectMessage connectMessage;
@@ -63,34 +60,16 @@ public class SqlConnection implements Closeable {
String database,
Map<String, Object> extraParams)
throws ClassNotFoundException, SQLException {
- // Handle mysql security vulnerabilities
- validateParams(extraParams);
- connectMessage = new ConnectMessage(host, port, username, password, extraParams);
- conn = getDBConnection(connectMessage, database);
- // Try to create statement
- Statement statement = conn.createStatement();
- statement.close();
- }
-
- /**
- * Handle mysql security vulnerabilities
- *
- * @param extraParams
- */
- private void validateParams(Map<String, Object> extraParams) {
- if (extraParams == null) {
- return;
- }
-
// security check
- SecurityUtils.checkJdbcSecurity(extraParams);
-
- // append force params
+ SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams);
SecurityUtils.appendMysqlForceParams(extraParams);
- // print extraParams
- String logStr = SecurityUtils.parseParamsMapToMysqlParamUrl(extraParams);
- LOG.info("mysql metadata url extraParams: {}", logStr);
+ connectMessage =
+ new ConnectMessage(host.trim(), port, username.trim(), password.trim(), extraParams);
+ conn = getDBConnection(connectMessage, database.trim());
+ // Try to create statement
+ Statement statement = conn.createStatement();
+ statement.close();
}
public List<String> getAllDatabases() throws SQLException {
@@ -230,6 +209,7 @@ public class SqlConnection implements Closeable {
if (!connectMessage.extraParams.isEmpty()) {
url += "?" + extraParamString;
}
+ LOG.info("jdbc connection url: {}", url);
return DriverManager.getConnection(url, connectMessage.username, connectMessage.password);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org