You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/08/17 09:33:08 UTC
[iotdb] branch master updated: [IOTDB-1566] Do not restrict
concurrent write partitions (#3767)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0dc0eee [IOTDB-1566] Do not restrict concurrent write partitions (#3767)
0dc0eee is described below
commit 0dc0eee6515c2edc020f3e650b8d10b1215f2a1e
Author: Steve Yurong Su (宇荣) <ro...@apache.org>
AuthorDate: Tue Aug 17 04:32:45 2021 -0500
[IOTDB-1566] Do not restrict concurrent write partitions (#3767)
---
.../src/assembly/resources/conf/iotdb-engine.properties | 2 ++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 +++++++
.../db/engine/storagegroup/StorageGroupProcessor.java | 16 +---------------
4 files changed, 11 insertions(+), 16 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 14c1330..cab6136 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -884,6 +884,8 @@ timestamp_precision=ms
# Datatype: long
# partition_interval=604800
+# concurrent_writing_time_partition=500
+
# admin username, default is root
# Datatype: string
# admin_name=root
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 89cc759..1047670 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -620,7 +620,7 @@ public class IoTDBConfig {
private String kerberosPrincipal = "your principal";
/** the num of memtable in each storage group */
- private int concurrentWritingTimePartition = 1;
+ private int concurrentWritingTimePartition = 500;
/** the default fill interval in LinearFill and PreviousFill, -1 means infinite past time */
private int defaultFillInterval = -1;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 599bc78..174f2fc 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -640,6 +640,13 @@ public class IoTDBDescriptor {
Long.parseLong(
properties.getProperty("default_ttl", String.valueOf(conf.getDefaultTTL()))));
+ // the num of memtables in each storage group
+ conf.setConcurrentWritingTimePartition(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_writing_time_partition",
+ String.valueOf(conf.getConcurrentWritingTimePartition()))));
+
conf.setTimeIndexLevel(
properties.getProperty("time_index_level", String.valueOf(conf.getTimeIndexLevel())));
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index c2f6ca4..36407db 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1238,21 +1238,7 @@ public class StorageGroupProcessor {
TsFileProcessor res = tsFileProcessorTreeMap.get(timeRangeId);
if (null == res) {
- // we have to remove oldest processor to control the num of the memtables
- // TODO: use a method to control the number of memtables
- if (tsFileProcessorTreeMap.size()
- >= IoTDBDescriptor.getInstance().getConfig().getConcurrentWritingTimePartition()) {
- Map.Entry<Long, TsFileProcessor> processorEntry = tsFileProcessorTreeMap.firstEntry();
- logger.info(
- "will close a {} TsFile because too many active partitions ({} > {}) in the storage group {},",
- sequence,
- tsFileProcessorTreeMap.size(),
- IoTDBDescriptor.getInstance().getConfig().getConcurrentWritingTimePartition(),
- logicalStorageGroupName);
- asyncCloseOneTsFileProcessor(sequence, processorEntry.getValue());
- }
-
- // build new processor
+ // build new processor, memory control module will control the number of memtables
res = newTsFileProcessor(sequence, timeRangeId);
tsFileProcessorTreeMap.put(timeRangeId, res);
tsFileManagement.add(res.getTsFileResource(), sequence);