You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/07/29 06:48:21 UTC
(iotdb) branch SortBufferSize-1.3 created (now 88a083b02b2)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch SortBufferSize-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 88a083b02b2 Change sort_buffer_size_in_bytes default value
This branch includes the following new commits:
new 88a083b02b2 Change sort_buffer_size_in_bytes default value
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Change sort_buffer_size_in_bytes default value
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch SortBufferSize-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 88a083b02b2dd17e19171f9570097ab4266b2b52
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Tue Jul 29 14:16:59 2025 +0800
Change sort_buffer_size_in_bytes default value
(cherry picked from commit 2459e5211ce7ded5a355ff6d96abbdcda10700f9)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++++++++++++++++++----
.../conf/iotdb-system.properties.template | 6 +++--
3 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 007f45755ab..531fa7f543f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -452,7 +452,7 @@ public class IoTDBConfig {
private volatile boolean enableAutoRepairCompaction = true;
/** The buffer for sort operation */
- private long sortBufferSize = 1024 * 1024L;
+ private long sortBufferSize = 32 * 1024 * 1024L;
/**
* The strategy of inner space compaction task. There are just one inner space compaction strategy
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7fbd9ce2ed7..7c61bab1a1f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1080,11 +1080,8 @@ public class IoTDBDescriptor {
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
// The buffer for sort operator to calculate
- conf.setSortBufferSize(
- Long.parseLong(
- properties
- .getProperty("sort_buffer_size_in_bytes", Long.toString(conf.getSortBufferSize()))
- .trim()));
+
+ loadSortBuffer(properties);
// tmp filePath for sort operator
conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir()));
@@ -1109,6 +1106,23 @@ public class IoTDBDescriptor {
loadTrustedUriPattern(properties);
}
+ private void loadSortBuffer(TrimProperties properties) {
+ long defaultValue = calculateDefaultSortBufferSize();
+ long sortBufferSize =
+ Long.parseLong(
+ properties.getProperty("sort_buffer_size_in_bytes", Long.toString(defaultValue)));
+ if (sortBufferSize <= 0) {
+ sortBufferSize = defaultValue;
+ }
+ // The buffer for sort operator to calculate
+ conf.setSortBufferSize(sortBufferSize);
+ }
+
+ public static long calculateDefaultSortBufferSize() {
+ return Math.min(
+ 32 * 1024 * 1024L, conf.getAllocateMemoryForOperators() / conf.getQueryThreadCount() / 2);
+ }
+
private void reloadConsensusProps(TrimProperties properties) throws IOException {
loadIoTConsensusProps(properties);
loadPipeConsensusProps(properties);
@@ -2042,6 +2056,9 @@ public class IoTDBDescriptor {
properties.getProperty(
"tvlist_sort_threshold",
ConfigurationFileUtils.getConfigurationDefaultValue("tvlist_sort_threshold"))));
+
+ // sort_buffer_size_in_bytes
+ loadSortBuffer(properties);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index f2fa57164f5..70bd7e15113 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1061,9 +1061,11 @@ mode_map_size_threshold=10000
batch_size=100000
# The memory for external sort in sort operator, when the data size is smaller than sort_buffer_size_in_bytes, the sort operator will use in-memory sort.
-# effectiveMode: restart
+# if sort_buffer_size_in_bytes <= 0, default value will be used, default value = min(32MB, memory for query operators / query_thread_count / 2)
+# if sort_buffer_size_in_bytes > 0, the specified value will be used.
+# effectiveMode: hot_reload
# Datatype: long
-sort_buffer_size_in_bytes=1048576
+sort_buffer_size_in_bytes=0
# The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged.
# effectiveMode: hot_reload