You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/03/09 10:06:59 UTC
[iotdb] branch seprate_binary_chunk_size created (now 3c8566cb8b)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch seprate_binary_chunk_size
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 3c8566cb8b Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
This branch includes the following new commits:
new 3c8566cb8b Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch seprate_binary_chunk_size
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3c8566cb8ba80610708ce270963aeb61478cdd2b
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Mar 9 18:06:42 2023 +0800
Seprate TargetMemoryBinaryChunkSize and TargetChunkSize configurations
---
docs/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++
docs/zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++
.../src/assembly/resources/conf/iotdb-common.properties | 7 ++++++-
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++
.../apache/iotdb/db/utils/datastructure/AlignedTVList.java | 4 ++--
.../org/apache/iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../java/org/apache/iotdb/db/utils/datastructure/TVList.java | 4 ++--
8 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index 5e6538fe9c..965508893c 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -754,6 +754,15 @@ Different configuration parameters take effect in the following three ways:
|Default| 100000 |
|Effective| After restarting system |
+* target\_memory\_binary\_chunk\_size
+
+|Name| target\_memory\_binary\_chunk\_size |
+|:---:|:-----------------------------------------------------------------------------------------------------------|
+|Description| When a Binary type of timeseries's size in a memtable reaches this threshold, flush this memtable to disk. |
+|Type| int64 |
+|Default| 1048576 |
+|Effective| After restarting system |
+
* flush\_thread\_count
|Name| flush\_thread\_count |
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 78883d6711..544e6454b2 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -802,6 +802,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|默认值| 100000 |
|改后生效方式| 重启服务生效 |
+* target\_memory\_binary\_chunk\_size
+
+|名字| target\_memory\_binary\_chunk\_size |
+|:---:|:----------------------------------------------|
+|描述| Memtable 中当一个 Binary 类型的序列内存大小超过这个阈值，触发 flush |
+|类型| int64 |
+|默认值| 1048576 |
+|改后生效方式| 重启服务生效 |
+
* flush\_thread\_count
|名字| flush\_thread\_count |
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 05a496dc89..7783db1da0 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -501,6 +501,11 @@ cluster_name=defaultCluster
# Datatype: int
# avg_series_point_number_threshold=100000
+# When a Binary type of timeseries's size in a memtable reaches this threshold, flush this memtable to disk.
+# default is 1MB
+# Datatype: long, Unit: byte
+# target_memory_binary_chunk_size=1048576
+
# How many threads can concurrently flush. When <= 0, use CPU core number.
# Datatype: int
# flush_thread_count=0
@@ -572,7 +577,7 @@ cluster_name=defaultCluster
# Datatype: long, Unit: byte
# target_compaction_file_size=1073741824
-# The target chunk size in compaction and when memtable reaches this threshold, flush the memtable to disk.
+# The target chunk size in compaction
# default is 1MB
# Datatype: long, Unit: byte
# target_chunk_size=1048576
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 70a0b0f3da..400fac2279 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -387,6 +387,9 @@ public class IoTDBConfig {
/** When average series point number reaches this, flush the memtable to disk */
private int avgSeriesPointNumberThreshold = 100000;
+ /** The target memory Binary chunk size in TVList. */
+ private long targetMemoryBinaryChunkSize = 1048576L;
+
/** Enable inner space compaction for sequence files */
private boolean enableSeqSpaceCompaction = true;
@@ -1966,6 +1969,14 @@ public class IoTDBConfig {
this.avgSeriesPointNumberThreshold = avgSeriesPointNumberThreshold;
}
+ public long getTargetMemoryBinaryChunkSize() {
+ return targetMemoryBinaryChunkSize;
+ }
+
+ public void setTargetMemoryBinaryChunkSize(long targetMemoryBinaryChunkSize) {
+ this.targetMemoryBinaryChunkSize = targetMemoryBinaryChunkSize;
+ }
+
public long getCrossCompactionFileSelectionTimeBudget() {
return crossCompactionFileSelectionTimeBudget;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 0432754ab7..894cd88514 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -392,6 +392,12 @@ public class IoTDBDescriptor {
"avg_series_point_number_threshold",
Integer.toString(conf.getAvgSeriesPointNumberThreshold()))));
+ conf.setTargetMemoryBinaryChunkSize(
+ Long.parseLong(
+ properties.getProperty(
+ "target_memory_binary_chunk_size",
+ Long.toString(conf.getTargetMemoryBinaryChunkSize()))));
+
conf.setCheckPeriodWhenInsertBlocked(
Integer.parseInt(
properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index dfba829696..ec1c97ef27 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -186,7 +186,7 @@ public abstract class AlignedTVList extends TVList {
columnValue != null
? getBinarySize((Binary) columnValue)
: getBinarySize(Binary.EMPTY_VALUE);
- if (memoryBinaryChunkSize[i] >= targetChunkSize) {
+ if (memoryBinaryChunkSize[i] >= targetMemoryBinaryChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
@@ -773,7 +773,7 @@ public abstract class AlignedTVList extends TVList {
memoryBinaryChunkSize[i] +=
arrayT[elementIndex + i1] != null ? getBinarySize(arrayT[elementIndex + i1]) : 0;
}
- if (memoryBinaryChunkSize[i] > targetChunkSize) {
+ if (memoryBinaryChunkSize[i] > targetMemoryBinaryChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index 1e644e41a6..49ec892213 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -99,7 +99,7 @@ public abstract class BinaryTVList extends TVList {
@Override
public boolean reachMaxChunkSizeThreshold() {
- return memoryBinaryChunkSize >= targetChunkSize;
+ return memoryBinaryChunkSize >= targetMemoryBinaryChunkSize;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 01d68e445a..1c0c70e521 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -49,8 +49,8 @@ public abstract class TVList implements WALEntryValue {
protected static final int SMALL_ARRAY_LENGTH = 32;
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent";
- protected static final long targetChunkSize =
- IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ protected static final long targetMemoryBinaryChunkSize =
+ IoTDBDescriptor.getInstance().getConfig().getTargetMemoryBinaryChunkSize();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;