You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/10 15:28:37 UTC
[doris] branch branch-1.2-lts updated: Revert "[branch-1.2][improvement](jdbc) Set the JDBC connection timeout to be configurable (#20595)"
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 2fceca2e7b Revert "[branch-1.2][improvement](jdbc) Set the JDBC connection timeout to be configurable (#20595)"
2fceca2e7b is described below
commit 2fceca2e7bcf3f595c3b8ce4959ee1ebbf6388ea
Author: morningman <mo...@163.com>
AuthorDate: Sat Jun 10 23:28:21 2023 +0800
Revert "[branch-1.2][improvement](jdbc) Set the JDBC connection timeout to be configurable (#20595)"
This reverts commit 530fcb76f408abd4885ba79cbc4fcbfa13ad2248.
---
be/src/common/config.h | 12 --------
be/src/util/jni-util.cpp | 36 ++--------------------
bin/start_be.sh | 16 ++++++----
conf/be.conf | 4 +--
.../main/java/org/apache/doris/common/Config.java | 12 --------
.../org/apache/doris/external/jdbc/JdbcClient.java | 6 ++--
.../java/org/apache/doris/udf/JdbcExecutor.java | 17 +++++-----
7 files changed, 24 insertions(+), 79 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 4d4f7e0c14..1f437a4d58 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -910,18 +910,6 @@ CONF_Int32(segcompaction_small_threshold, "1048576");
// enable java udf and jdbc scannode
CONF_Bool(enable_java_support, "true");
-// jdbc connect init pool size
-CONF_Int32(jdbc_init_pool_size, "1");
-
-// jdbc connect maximum number of active connections
-CONF_Int32(jdbc_max_pool_size, "100");
-
-// jdbc connect maximum idel time
-CONF_Int32(jdbc_max_idel_time, "300000");
-
-// jdbc connect maximum wait time
-CONF_Int32(jdbc_max_wait_time, "5000");
-
// Set config randomly to check more issues in github workflow
CONF_Bool(enable_fuzzy_mode, "false");
diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp
index 16756d4458..2b1b82983e 100644
--- a/be/src/util/jni-util.cpp
+++ b/be/src/util/jni-util.cpp
@@ -72,30 +72,6 @@ const std::string GetDorisJNIClasspathOption() {
}
}
-struct JdbcOptions {
- int init_pool;
- int max_pool;
- int max_idel_time;
- int max_wait_time;
-
- JdbcOptions() {
- init_pool = config::jdbc_init_pool_size;
- max_pool = config::jdbc_max_pool_size;
- max_idel_time = config::jdbc_max_idel_time;
- max_wait_time = config::jdbc_max_wait_time;
- }
-
- std::string toString() const {
- std::stringstream jdbc_str;
- jdbc_str << " "
- << "-DJDBC_INIT_POOL=" << init_pool << " "
- << "-DJDBC_MAX_POOL=" << max_pool << " "
- << "-DJDBC_MAX_IDEL_TIME=" << max_idel_time << " "
- << "-DJDBC_MAX_WAIT_TIME=" << max_wait_time;
- return jdbc_str.str();
- }
-};
-
[[maybe_unused]] void SetEnvIfNecessary() {
const auto* doris_home = getenv("DORIS_HOME");
DCHECK(doris_home) << "Environment variable DORIS_HOME is not set.";
@@ -105,13 +81,9 @@ struct JdbcOptions {
fmt::format("{}/conf:{}", doris_home, GetDorisJNIDefaultClasspath());
setenv("CLASSPATH", classpath.c_str(), 0);
- JdbcOptions jdbcOpts;
- std::string jdbc_opts_str = jdbcOpts.toString();
-
// LIBHDFS_OPTS
setenv("LIBHDFS_OPTS",
- fmt::format("-Djava.library.path={}/lib/hadoop_hdfs/native", getenv("DORIS_HOME"),
- jdbc_opts_str)
+ fmt::format("-Djava.library.path={}/lib/hadoop_hdfs/native", getenv("DORIS_HOME"))
.c_str(),
0);
}
@@ -123,16 +95,12 @@ struct JdbcOptions {
if (rv == 0) {
std::vector<std::string> options;
- JdbcOptions jdbcOpts;
- std::string jdbc_opts_str = jdbcOpts.toString();
-
char* java_opts = getenv("JAVA_OPTS");
if (java_opts == nullptr) {
options = {
GetDorisJNIClasspathOption(), fmt::format("-Xmx{}", "1g"),
fmt::format("-DlogPath={}/log/jni.log", getenv("DORIS_HOME")),
fmt::format("-Dsun.java.command={}", "DorisBE"), "-XX:-CriticalJNINatives",
- jdbc_opts_str,
#ifdef __APPLE__
// On macOS, we should disable MaxFDLimit, otherwise the RLIMIT_NOFILE
// will be assigned the minimum of OPEN_MAX (10240) and rlim_cur (See src/hotspot/os/bsd/os_bsd.cpp)
@@ -142,7 +110,7 @@ struct JdbcOptions {
#endif
};
} else {
- std::istringstream stream(java_opts + jdbc_opts_str);
+ std::istringstream stream(java_opts);
options = std::vector<std::string>(std::istream_iterator<std::string> {stream},
std::istream_iterator<std::string>());
options.push_back(GetDorisJNIClasspathOption());
diff --git a/bin/start_be.sh b/bin/start_be.sh
index ab04ff18f9..b36c1569a4 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -269,16 +269,20 @@ java_version="$(
)"
CUR_DATE=$(date +%Y%m%d-%H%M%S)
+LOG_PATH="-DlogPath=${DORIS_HOME}/log/jni.log"
+COMMON_OPTS="-Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
+JDBC_OPTS="-DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDEL_TIME=300000"
+
if [[ "${java_version}" -gt 8 ]]; then
- if [[ -z ${JAVA_OPTS_FOR_JDK_9} ]]; then
- JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=${DORIS_HOME}/log/jni.log -Xlog:gc:${DORIS_HOME}/log/be.gc.log.${CUR_DATE} -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
- fi
- final_java_opt="${JAVA_OPTS_FOR_JDK_9}"
-else
if [[ -z ${JAVA_OPTS} ]]; then
- JAVA_OPTS="-Xmx1024m -DlogPath=${DORIS_HOME}/log/jni.log -Xloggc:${DORIS_HOME}/log/be.gc.log.${CUR_DATE} -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
+ JAVA_OPTS="-Xmx1024m ${LOG_PATH} -Xloggc:${DORIS_HOME}/log/be.gc.log.${CUR_DATE} ${COMMON_OPTS} ${JDBC_OPTS}"
fi
final_java_opt="${JAVA_OPTS}"
+else
+ if [[ -z ${JAVA_OPTS_FOR_JDK_9} ]]; then
+ JAVA_OPTS_FOR_JDK_9="-Xmx1024m ${LOG_PATH} -Xlog:gc:${DORIS_HOME}/log/be.gc.log.${CUR_DATE} ${COMMON_OPTS} ${JDBC_OPTS}"
+ fi
+ final_java_opt="${JAVA_OPTS_FOR_JDK_9}"
fi
if [[ "${MACHINE_OS}" == "Darwin" ]]; then
diff --git a/conf/be.conf b/conf/be.conf
index 87e730bb93..dcb9135813 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -18,10 +18,10 @@
PPROF_TMPDIR="$DORIS_HOME/log/"
CUR_DATE=`date +%Y%m%d-%H%M%S`
-JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
+JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDEL_TIME=300000"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
+JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDEL_TIME=300000"
# since 1.2, the JAVA_HOME need to be set to run BE process.
# JAVA_HOME=/path/to/jdk/
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index ee17638a34..796c5eca0f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -133,18 +133,6 @@ public class Config extends ConfigBase {
@ConfField
public static String jdbc_drivers_dir = System.getenv("DORIS_HOME") + "/jdbc_drivers";
- // jdbc catalog initial number of connections
- @ConfField(mutable = true, masterOnly = true)
- public static int jdbc_init_pool_size = 1;
-
- // jdbc catalog maximum number of active connections
- @ConfField(mutable = true, masterOnly = true)
- public static int jdbc_max_pool_size = 100;
-
- // jdbc catalog connection timeout time, default 5s, do not set more than 30s
- @ConfField(mutable = true, masterOnly = true)
- public static int jdbc_max_wait_time = 5000;
-
/**
* The default parallelism of the load execution plan
* on a single node when the broker load is submitted
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
index 521695baa3..3f635fc762 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
@@ -96,8 +96,8 @@ public class JdbcClient {
dataSource.setUsername(jdbcUser);
dataSource.setPassword(password);
dataSource.setMinIdle(1);
- dataSource.setInitialSize(Config.jdbc_init_pool_size);
- dataSource.setMaxActive(Config.jdbc_max_pool_size);
+ dataSource.setInitialSize(1);
+ dataSource.setMaxActive(100);
dataSource.setTimeBetweenEvictionRunsMillis(600000);
dataSource.setMinEvictableIdleTimeMillis(300000);
// set connection timeout to 5s.
@@ -105,7 +105,7 @@ public class JdbcClient {
// Because when querying information_schema db, BE will call thrift rpc(default timeout is 30s)
// to FE to get schema info, and may create connection here, if we set it too long and the url is invalid,
// it may cause the thrift rpc timeout.
- dataSource.setMaxWait(Config.jdbc_max_wait_time);
+ dataSource.setMaxWait(5000);
} catch (MalformedURLException e) {
throw new JdbcClientException("MalformedURLException to load class about " + driverUrl, e);
} finally {
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
index dba12e145b..2dfb81020d 100644
--- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
@@ -72,11 +72,10 @@ public class JdbcExecutor {
private int curBlockRows = 0;
private static final byte[] emptyBytes = new byte[0];
private DruidDataSource druidDataSource = null;
- private int initPoolSize;
+ private int minPoolSize;
private int maxPoolSize;
private int minIdleSize;
private int maxIdelTime;
- private int maxWaitTime;
public JdbcExecutor(byte[] thriftParams) throws Exception {
TJdbcExecutorCtorParams request = new TJdbcExecutorCtorParams();
@@ -86,16 +85,14 @@ public class JdbcExecutor {
} catch (TException e) {
throw new InternalException(e.getMessage());
}
- initPoolSize = Integer.valueOf(System.getProperty("JDBC_INIT_POOL", "1"));
+ minPoolSize = Integer.valueOf(System.getProperty("JDBC_MIN_POOL", "1"));
maxPoolSize = Integer.valueOf(System.getProperty("JDBC_MAX_POOL", "100"));
maxIdelTime = Integer.valueOf(System.getProperty("JDBC_MAX_IDEL_TIME", "300000"));
- maxWaitTime = Integer.valueOf(System.getProperty("JDBC_MAX_WAIT_TIME", "5000"));
- minIdleSize = initPoolSize > 0 ? 1 : 0;
- LOG.info("JdbcExecutor set initPoolSize = " + initPoolSize
+ minIdleSize = minPoolSize > 0 ? 1 : 0;
+ LOG.info("JdbcExecutor set minPoolSize = " + minPoolSize
+ ", maxPoolSize = " + maxPoolSize
+ ", maxIdelTime = " + maxIdelTime
- + ", minIdleSize = " + minIdleSize
- + ", minWaitTime = " + maxWaitTime);
+ + ", minIdleSize = " + minIdleSize);
init(request.driver_path, request.statement, request.batch_size, request.jdbc_driver_class,
request.jdbc_url, request.jdbc_user, request.jdbc_password, request.op, request.table_type);
}
@@ -277,9 +274,9 @@ public class JdbcExecutor {
ds.setUsername(jdbcUser);
ds.setPassword(jdbcPassword);
ds.setMinIdle(minIdleSize);
- ds.setInitialSize(initPoolSize);
+ ds.setInitialSize(minPoolSize);
ds.setMaxActive(maxPoolSize);
- ds.setMaxWait(maxWaitTime);
+ ds.setMaxWait(5000);
ds.setTestWhileIdle(true);
ds.setTestOnBorrow(false);
setValidationQuery(ds, tableType);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org