You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/07/22 03:27:31 UTC
(iotdb) branch SortBufferSize created (now 18cc581b6f1)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch SortBufferSize
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 18cc581b6f1 Change sort_buffer_size_in_bytes default value
This branch includes the following new commits:
new 18cc581b6f1 Change sort_buffer_size_in_bytes default value
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Change sort_buffer_size_in_bytes default value
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch SortBufferSize
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 18cc581b6f1b4c00d2d086a8929949df0d698d14
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Jul 22 11:27:14 2025 +0800
Change sort_buffer_size_in_bytes default value
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 28 ++++++++++++++++++----
.../conf/iotdb-system.properties.template | 6 +++--
3 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b751c150585..2493ad221ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -73,6 +73,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.CONSENSUS_FOLDER_NAME;
import static org.apache.iotdb.commons.conf.IoTDBConstant.DELETION_FOLDER_NAME;
import static org.apache.iotdb.commons.conf.IoTDBConstant.OBJECT_STORAGE_DIR;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PIPE_FOLDER_NAME;
+import static org.apache.iotdb.db.conf.IoTDBDescriptor.calculateDefaultSortBufferSize;
import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
public class IoTDBConfig {
@@ -415,7 +416,8 @@ public class IoTDBConfig {
private volatile boolean enableAutoRepairCompaction = true;
/** The buffer for sort operation */
- private long sortBufferSize = 1024 * 1024L;
+ private long sortBufferSize =
+ calculateDefaultSortBufferSize(IoTDBDescriptor.getInstance().getMemoryConfig());
/**
* The strategy of inner space compaction task. There are just one inner space compaction strategy
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 34501ffff69..7a6bfc8aa58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1027,10 +1027,7 @@ public class IoTDBDescriptor {
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
// The buffer for sort operator to calculate
- conf.setSortBufferSize(
- Long.parseLong(
- properties.getProperty(
- "sort_buffer_size_in_bytes", Long.toString(conf.getSortBufferSize()))));
+ loadSortBuffer(properties);
// tmp filePath for sort operator
conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir()));
@@ -1093,6 +1090,26 @@ public class IoTDBDescriptor {
loadTrustedUriPattern(properties);
}
+ private void loadSortBuffer(TrimProperties properties) {
+ long defaultValue = calculateDefaultSortBufferSize(memoryConfig);
+ long sortBufferSize =
+ Long.parseLong(
+ properties.getProperty("sort_buffer_size_in_bytes", Long.toString(defaultValue)));
+ if (sortBufferSize <= 0) {
+ sortBufferSize = defaultValue;
+ }
+ // The buffer for sort operator to calculate
+ conf.setSortBufferSize(sortBufferSize);
+ }
+
+ public static long calculateDefaultSortBufferSize(DataNodeMemoryConfig memoryConfig) {
+ return Math.min(
+ 32 * 1024 * 1024L,
+ memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes()
+ / memoryConfig.getQueryThreadCount()
+ / 2);
+ }
+
private void reloadConsensusProps(TrimProperties properties) throws IOException {
loadIoTConsensusProps(properties);
loadIoTConsensusV2Props(properties);
@@ -2064,6 +2081,9 @@ public class IoTDBDescriptor {
properties.getProperty(
"tvlist_sort_threshold",
ConfigurationFileUtils.getConfigurationDefaultValue("tvlist_sort_threshold"))));
+
+ // sort_buffer_size_in_bytes
+ loadSortBuffer(properties);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 28af70751ba..67811dacadb 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1070,9 +1070,11 @@ mode_map_size_threshold=10000
batch_size=100000
# The memory for external sort in sort operator, when the data size is smaller than sort_buffer_size_in_bytes, the sort operator will use in-memory sort.
-# effectiveMode: restart
+# if sort_buffer_size_in_bytes <= 0, default value will be used, default value = min(32MB, memory for query operators / query_thread_count / 2)
+# if sort_buffer_size_in_bytes > 0, the specified value will be used.
+# effectiveMode: hot_reload
# Datatype: long
-sort_buffer_size_in_bytes=1048576
+sort_buffer_size_in_bytes=0
# The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged.
# effectiveMode: hot_reload