You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/01/29 09:31:46 UTC
[kylin] 04/06: support clikchouse jdbc url ext config
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3c84cd06ba6f43b9b2af1f07514bf1ae9a854f9f
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Sat Jan 14 14:42:16 2023 +0800
support clikchouse jdbc url ext config
Co-authored-by: chenzhx <ch...@apache.io>
---
.../org/apache/kylin/common/KylinConfigBase.java | 12 +++++++++
.../kyligence/kap/clickhouse/job/ClickHouse.java | 31 ++++++++++++++--------
.../kap/clickhouse/job/ClickHouseTest.java | 10 +++++--
3 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bebd3f403f..95fa963d5e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3621,6 +3621,18 @@ public abstract class KylinConfigBase implements Serializable {
return Long.parseLong(getOptional("kylin.second-storage.wait-index-build-second", "10"));
}
+ public String getSecondStorageJDBCKeepAliveTimeout() {
+ return getOptional("kylin.second-storage.jdbc-keep-alive-timeout", "600000");
+ }
+
+ public String getSecondStorageJDBCSocketTimeout() {
+ return getOptional("kylin.second-storage.jdbc-socket-timeout", "600000");
+ }
+
+ public String getSecondStorageJDBCExtConfig() {
+ return getOptional("kylin.second-storage.jdbc-ext-config", "connect_timeout=3");
+ }
+
public long getRoutineOpsTaskTimeOut() {
return TimeUtil.timeStringAs(getOptional("kylin.metadata.ops-cron-timeout", "4h"), TimeUnit.MILLISECONDS);
}
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
index 9664bde2e7..9822425ff8 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
@@ -18,12 +18,6 @@
package io.kyligence.kap.clickhouse.job;
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-import lombok.val;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-
import java.io.Closeable;
import java.sql.Connection;
import java.sql.Date;
@@ -44,6 +38,14 @@ import java.util.Objects;
import java.util.Properties;
import java.util.function.Function;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kylin.common.KylinConfig;
+
+import lombok.Getter;
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Getter
public class ClickHouse implements Closeable {
@@ -113,18 +115,25 @@ public class ClickHouse implements Closeable {
base.append('?');
List<String> paramList = new ArrayList<>();
param.forEach((name, value) -> {
- if (!ClickHouse.EXT_CONFIG.equals(name)){
+ if (ClickHouse.SOCKET_TIMEOUT.equals(name)) {
+ value = getNoEmptyValue(KylinConfig.getInstanceFromEnv().getSecondStorageJDBCSocketTimeout(), value);
+ }
+ if (ClickHouse.KEEP_ALIVE_TIMEOUT.equals(name)) {
+ value = getNoEmptyValue(KylinConfig.getInstanceFromEnv().getSecondStorageJDBCKeepAliveTimeout(), value);
+ }
paramList.add(name + "=" + value);
- }
});
base.append(String.join("&", paramList));
- if(param.get(ClickHouse.EXT_CONFIG) != null) {
- base.append("&").append(param.get(ClickHouse.EXT_CONFIG));
- }
+ String extConfig = KylinConfig.getInstanceFromEnv().getSecondStorageJDBCExtConfig();
+ base.append("&").append(extConfig);
}
return base.toString();
}
+ private static String getNoEmptyValue(String value1, String value2) {
+ return StringUtils.isEmpty(value1) ? value2 : value1;
+ }
+
private void logSql(String sql) {
log.info("Execute SQL '{}' on [{}]", sql, shardName);
}
diff --git a/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/ClickHouseTest.java b/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/ClickHouseTest.java
index b3d8e27c04..60ca087f0e 100644
--- a/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/ClickHouseTest.java
+++ b/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/ClickHouseTest.java
@@ -23,6 +23,7 @@ import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
import io.kyligence.kap.secondstorage.config.ClusterInfo;
import io.kyligence.kap.secondstorage.config.Node;
import lombok.val;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -33,10 +34,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class ClickHouseTest {
+public class ClickHouseTest extends NLocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
+ createTestMetadata();
initNodeHelper();
}
@@ -66,7 +68,8 @@ public class ClickHouseTest {
@Test
public void extractParam() {
val param = ClickHouse.extractParam(SecondStorageNodeHelper.resolve("node01"));
- Assert.assertEquals(2, param.size());
+ Assert.assertEquals(3, param.size());
+ Assert.assertEquals("3", param.get("connect_timeout"));
val param2 = ClickHouse.extractParam(SecondStorageNodeHelper.resolve("node03"));
Assert.assertEquals(0, param2.size());
}
@@ -82,5 +85,8 @@ public class ClickHouseTest {
Assert.assertEquals(properties.get(ClickHouse.SOCKET_TIMEOUT), cluster.getSocketTimeout());
Assert.assertEquals(properties.get(ClickHouse.USER), node.getUser());
Assert.assertEquals(properties.get(ClickHouse.PASSWORD), node.getPassword());
+
+ String url = ClickHouse.buildUrl(node.getIp(), node.getPort(), properties);
+ Assert.assertEquals("jdbc:clickhouse://127.0.0.1:9000?socket_timeout=600000&keepAliveTimeout=600000&password=123456&user=default&connect_timeout=3", url);
}
}
\ No newline at end of file