You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/03/09 10:07:00 UTC
[iotdb] 01/01: Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch seprate_binary_chunk_size
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3c8566cb8ba80610708ce270963aeb61478cdd2b
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Mar 9 18:06:42 2023 +0800
Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
---
docs/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++
docs/zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++
.../src/assembly/resources/conf/iotdb-common.properties | 7 ++++++-
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++
.../apache/iotdb/db/utils/datastructure/AlignedTVList.java | 4 ++--
.../org/apache/iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../java/org/apache/iotdb/db/utils/datastructure/TVList.java | 4 ++--
8 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index 5e6538fe9c..965508893c 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -754,6 +754,15 @@ Different configuration parameters take effect in the following three ways:
|Default| 100000 |
|Effective| After restarting system |
+* target\_memory\_binary\_chunk\_size
+
+|Name| target\_memory\_binary\_chunk\_size |
+|:---:|:-----------------------------------------------------------------------------------------------------------|
+|Description| When a Binary type of timeseries's size in a memtable reaches this threshold, flush this memtable to disk. |
+|Type| int64 |
+|Default| 1048576 |
+|Effective| After restarting system |
+
* flush\_thread\_count
|Name| flush\_thread\_count |
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 78883d6711..544e6454b2 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -802,6 +802,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|默认值| 100000 |
|改后生效方式| 重启服务生效 |
+* target\_memory\_binary\_chunk\_size
+
+|名字| target\_memory\_binary\_chunk\_size |
+|:---:|:----------------------------------------------|
+|描述| Memtable 中当一个 Binary 类型的序列内存大小超过这个阈值，触发 flush |
+|类型| int64 |
+|默认值| 1048576 |
+|改后生效方式| 重启服务生效 |
+
* flush\_thread\_count
|名字| flush\_thread\_count |
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 05a496dc89..7783db1da0 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -501,6 +501,11 @@ cluster_name=defaultCluster
# Datatype: int
# avg_series_point_number_threshold=100000
+# When a Binary type of timeseries's size in a memtable reaches this threshold, flush this memtable to disk.
+# default is 1MB
+# Datatype: long, Unit: byte
+# target_memory_binary_chunk_size=1048576
+
# How many threads can concurrently flush. When <= 0, use CPU core number.
# Datatype: int
# flush_thread_count=0
@@ -572,7 +577,7 @@ cluster_name=defaultCluster
# Datatype: long, Unit: byte
# target_compaction_file_size=1073741824
-# The target chunk size in compaction and when memtable reaches this threshold, flush the memtable to disk.
+# The target chunk size in compaction
# default is 1MB
# Datatype: long, Unit: byte
# target_chunk_size=1048576
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 70a0b0f3da..400fac2279 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -387,6 +387,9 @@ public class IoTDBConfig {
/** When average series point number reaches this, flush the memtable to disk */
private int avgSeriesPointNumberThreshold = 100000;
+ /** The target memory Binary chunk size in TVList. */
+ private long targetMemoryBinaryChunkSize = 1048576L;
+
/** Enable inner space compaction for sequence files */
private boolean enableSeqSpaceCompaction = true;
@@ -1966,6 +1969,14 @@ public class IoTDBConfig {
this.avgSeriesPointNumberThreshold = avgSeriesPointNumberThreshold;
}
+ public long getTargetMemoryBinaryChunkSize() {
+ return targetMemoryBinaryChunkSize;
+ }
+
+ public void setTargetMemoryBinaryChunkSize(long targetMemoryBinaryChunkSize) {
+ this.targetMemoryBinaryChunkSize = targetMemoryBinaryChunkSize;
+ }
+
public long getCrossCompactionFileSelectionTimeBudget() {
return crossCompactionFileSelectionTimeBudget;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 0432754ab7..894cd88514 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -392,6 +392,12 @@ public class IoTDBDescriptor {
"avg_series_point_number_threshold",
Integer.toString(conf.getAvgSeriesPointNumberThreshold()))));
+ conf.setTargetMemoryBinaryChunkSize(
+ Long.parseLong(
+ properties.getProperty(
+ "target_memory_binary_chunk_size",
+ Long.toString(conf.getTargetMemoryBinaryChunkSize()))));
+
conf.setCheckPeriodWhenInsertBlocked(
Integer.parseInt(
properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index dfba829696..ec1c97ef27 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -186,7 +186,7 @@ public abstract class AlignedTVList extends TVList {
columnValue != null
? getBinarySize((Binary) columnValue)
: getBinarySize(Binary.EMPTY_VALUE);
- if (memoryBinaryChunkSize[i] >= targetChunkSize) {
+ if (memoryBinaryChunkSize[i] >= targetMemoryBinaryChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
@@ -773,7 +773,7 @@ public abstract class AlignedTVList extends TVList {
memoryBinaryChunkSize[i] +=
arrayT[elementIndex + i1] != null ? getBinarySize(arrayT[elementIndex + i1]) : 0;
}
- if (memoryBinaryChunkSize[i] > targetChunkSize) {
+ if (memoryBinaryChunkSize[i] > targetMemoryBinaryChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index 1e644e41a6..49ec892213 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -99,7 +99,7 @@ public abstract class BinaryTVList extends TVList {
@Override
public boolean reachMaxChunkSizeThreshold() {
- return memoryBinaryChunkSize >= targetChunkSize;
+ return memoryBinaryChunkSize >= targetMemoryBinaryChunkSize;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 01d68e445a..1c0c70e521 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -49,8 +49,8 @@ public abstract class TVList implements WALEntryValue {
protected static final int SMALL_ARRAY_LENGTH = 32;
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent";
- protected static final long targetChunkSize =
- IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ protected static final long targetMemoryBinaryChunkSize =
+ IoTDBDescriptor.getInstance().getConfig().getTargetMemoryBinaryChunkSize();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;