You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/11/27 17:34:45 UTC
[iotdb] branch master updated: [IOTDB-2061] Add max cached buffer size and max concurrent sub query parameter (#4486)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6a88895 [IOTDB-2061] Add max cached buffer size and max concurrent sub query parameter (#4486)
6a88895 is described below
commit 6a88895ce3b1afc300b0fda95a5010dd315f0ff0
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Sun Nov 28 01:34:13 2021 +0800
[IOTDB-2061] Add max cached buffer size and max concurrent sub query parameter (#4486)
---
server/src/assembly/resources/conf/iotdb-engine.properties | 7 +++----
server/src/assembly/resources/conf/iotdb-env.bat | 4 ++++
server/src/assembly/resources/conf/iotdb-env.sh | 7 ++++++-
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 ++++++------
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 9 +++------
.../org/apache/iotdb/db/query/pool/QueryTaskPoolManager.java | 10 ++++++++--
6 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 3e2da93..aad2f9d 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -284,10 +284,6 @@ timestamp_precision=ms
# Datatype: int
# concurrent_flush_thread=0
-# How many threads can concurrently query. When <= 0, use CPU core number.
-# Datatype: int
-# concurrent_query_thread=0
-
# whether take over the memory management by IoTDB rather than JVM when serializing memtable as bytes in memory
# (i.e., whether use ChunkBufferPool), value true, false
# Datatype: boolean
@@ -530,6 +526,9 @@ timestamp_precision=ms
# Datatype: int
# query_timeout_threshold=60000
+# The max number of sub query thread
+# max_concurrent_sub_query_thread=8
+
####################
### Metadata Cache Configuration
####################
diff --git a/server/src/assembly/resources/conf/iotdb-env.bat b/server/src/assembly/resources/conf/iotdb-env.bat
index 6a165ba..72a1237 100644
--- a/server/src/assembly/resources/conf/iotdb-env.bat
+++ b/server/src/assembly/resources/conf/iotdb-env.bat
@@ -103,9 +103,13 @@ for /f "tokens=1-3" %%j in ('java -version 2^>^&1') do (
@REM maximum direct memory size
set MAX_DIRECT_MEMORY_SIZE=%MAX_HEAP_SIZE%
+@REM Max cached buffer size, Note: unit can only be B!
+@REM which equals DIRECT_MEMORY_SIZE / threads number that may use direct memory / buffer cache size(IOV_MAX) depends on operating system
+set MAX_DIRECT_MEMORY_SIZE=%max_heap_size_in_mb% * 1024 * 1024 / 16 / 1024
set IOTDB_HEAP_OPTS=-Xmx%MAX_HEAP_SIZE% -Xms%HEAP_NEWSIZE% -Xlog:gc:"..\gc.log"
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:MaxDirectMemorySize=%MAX_DIRECT_MEMORY_SIZE%
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -Djdk.nio.maxCachedBufferSize=%MAX_CACHED_BUFFER_SIZE%
@REM You can put your env variable here
@REM set JAVA_HOME=%JAVA_HOME%
diff --git a/server/src/assembly/resources/conf/iotdb-env.sh b/server/src/assembly/resources/conf/iotdb-env.sh
index a62f99e..954c1ee 100755
--- a/server/src/assembly/resources/conf/iotdb-env.sh
+++ b/server/src/assembly/resources/conf/iotdb-env.sh
@@ -205,8 +205,12 @@ calculate_heap_sizes
#MAX_HEAP_SIZE="2G"
# Minimum heap size
#HEAP_NEWSIZE="2G"
-# maximum direct memory size
+# Maximum direct memory size
MAX_DIRECT_MEMORY_SIZE=${MAX_HEAP_SIZE}
+# Max cached buffer size, Note: unit can only be B!
+# which equals DIRECT_MEMORY_SIZE / threads number that may use direct memory / buffer cache size(IOV_MAX) depends on operating system
+MAX_CACHED_BUFFER_SIZE=`expr $max_heap_size_in_mb \* 1024 \* 1024 / 16 / 1024`
+echo $MAX_CACHED_BUFFER_SIZE
#true or false
#DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY (${IOTDB_CONF}/jmx.password and ${IOTDB_CONF}/jmx.access)
@@ -241,6 +245,7 @@ fi
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${HEAP_NEWSIZE}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${MAX_HEAP_SIZE}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${MAX_DIRECT_MEMORY_SIZE}"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
echo "Maximum memory allocation pool = ${MAX_HEAP_SIZE}B, initial memory allocation pool = ${HEAP_NEWSIZE}B"
echo "If you want to change this configuration, please check conf/iotdb-env.sh(Unix or OS X, if you use Windows, check conf/iotdb-env.bat)."
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 71f9da7..0ef9fbe 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -252,8 +252,8 @@ public class IoTDBConfig {
/** How many threads can concurrently flush. When <= 0, use CPU core number. */
private int concurrentFlushThread = Runtime.getRuntime().availableProcessors();
- /** How many threads can concurrently query. When <= 0, use CPU core number. */
- private int concurrentQueryThread = Runtime.getRuntime().availableProcessors();
+ /** How many threads can concurrently query. */
+ private int maxConcurrentSubQueryThread = 8;
/** How many threads can concurrently evaluate windows. When <= 0, use CPU core number. */
private int concurrentWindowEvaluationThread = Runtime.getRuntime().availableProcessors();
@@ -1159,12 +1159,12 @@ public class IoTDBConfig {
this.concurrentFlushThread = concurrentFlushThread;
}
- public int getConcurrentQueryThread() {
- return concurrentQueryThread;
+ public int getMaxConcurrentSubQueryThread() {
+ return maxConcurrentSubQueryThread;
}
- void setConcurrentQueryThread(int concurrentQueryThread) {
- this.concurrentQueryThread = concurrentQueryThread;
+ void setMaxConcurrentSubQueryThread(int maxConcurrentSubQueryThread) {
+ this.maxConcurrentSubQueryThread = maxConcurrentSubQueryThread;
}
public int getConcurrentWindowEvaluationThread() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 17deb09..84f8051 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -440,14 +440,11 @@ public class IoTDBDescriptor {
"index_buffer_size", Long.toString(conf.getIndexBufferSize()))));
// end: index parameter setting
- conf.setConcurrentQueryThread(
+ conf.setMaxConcurrentSubQueryThread(
Integer.parseInt(
properties.getProperty(
- "concurrent_query_thread", Integer.toString(conf.getConcurrentQueryThread()))));
-
- if (conf.getConcurrentQueryThread() <= 0) {
- conf.setConcurrentQueryThread(Runtime.getRuntime().availableProcessors());
- }
+ "max_concurrent_sub_query_thread",
+ Integer.toString(conf.getMaxConcurrentSubQueryThread()))));
conf.setmManagerCacheSize(
Integer.parseInt(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/pool/QueryTaskPoolManager.java b/server/src/main/java/org/apache/iotdb/db/query/pool/QueryTaskPoolManager.java
index fa284b7..41f662c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/pool/QueryTaskPoolManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/pool/QueryTaskPoolManager.java
@@ -32,7 +32,10 @@ public class QueryTaskPoolManager extends AbstractPoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(QueryTaskPoolManager.class);
private QueryTaskPoolManager() {
- int threadCnt = IoTDBDescriptor.getInstance().getConfig().getConcurrentQueryThread();
+ int threadCnt =
+ Math.min(
+ Runtime.getRuntime().availableProcessors(),
+ IoTDBDescriptor.getInstance().getConfig().getMaxConcurrentSubQueryThread());
pool = IoTDBThreadPoolFactory.newFixedThreadPool(threadCnt, ThreadName.QUERY_SERVICE.getName());
}
@@ -53,7 +56,10 @@ public class QueryTaskPoolManager extends AbstractPoolManager {
@Override
public void start() {
if (pool == null) {
- int threadCnt = IoTDBDescriptor.getInstance().getConfig().getConcurrentQueryThread();
+ int threadCnt =
+ Math.min(
+ Runtime.getRuntime().availableProcessors(),
+ IoTDBDescriptor.getInstance().getConfig().getMaxConcurrentSubQueryThread());
pool =
IoTDBThreadPoolFactory.newFixedThreadPool(threadCnt, ThreadName.QUERY_SERVICE.getName());
}