[iotdb] branch rel/0.12 updated: [To rel/0.12] Separate unseq file num in each level (#3189)

[qi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/rel/0.12 by this push:
     new 2522bd5  [To rel/0.12] Separate unseq file num in each level (#3189)
2522bd5 is described below

commit 2522bd52b252fa46c3251fc4c32817e982a90837
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Fri May 14 16:13:44 2021 +0800

    [To rel/0.12] Separate unseq file num in each level (#3189)
---
 .../src/assembly/resources/conf/iotdb-engine.properties |  7 +++++++
 .../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 17 +++++++++++++++++
 .../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java  |  6 ++++++
 .../iotdb/db/engine/compaction/TsFileManagement.java    | 12 ++++++++++++
 4 files changed, 42 insertions(+)

diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 54ae811..123db71 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -338,6 +338,13 @@ timestamp_precision=ms
 # The max num of unseq level.
 # unseq_level_num=1
 
+# Works when compaction_strategy is LEVEL_COMPACTION.
+# The max open file num in each unseq compaction task.
+# We use the unseq file num as the open file num
+# This parameters have to be much smaller than the permitted max open file num of each process controlled by operator system(65535 in most system)
+# Datatype: int
+# max_select_unseq_file_num_in_each_unseq_compaction=2000
+
 # Works when the compaction_strategy is LEVEL_COMPACTION.
 # When the average point number of chunks in the target file reaches this, merge the file to the top level.
 # During a merge, if a chunk with less number of points than this parameter, the chunk will be
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 7ae9755..8117433 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -323,6 +323,14 @@ public class IoTDBConfig {
   /** Works when the compaction_strategy is LEVEL_COMPACTION. The max num of unseq level. */
   private int unseqLevelNum = 1;
 
+  /**
+   * Works when compaction_strategy is LEVEL_COMPACTION. The max open file num in each unseq
+   * compaction task. We use the unseq file num as the open file num # This parameters have to be
+   * much smaller than the permitted max open file num of each process controlled by operator
+   * system(65535 in most system).
+   */
+  private int maxSelectUnseqFileNumInEachUnseqCompaction = 2000;
+
   /** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
   private boolean metaDataCacheEnable = true;
 
@@ -1465,6 +1473,15 @@ public class IoTDBConfig {
     this.unseqLevelNum = unseqLevelNum;
   }
 
+  public int getMaxSelectUnseqFileNumInEachUnseqCompaction() {
+    return maxSelectUnseqFileNumInEachUnseqCompaction;
+  }
+
+  public void setMaxSelectUnseqFileNumInEachUnseqCompaction(
+      int maxSelectUnseqFileNumInEachUnseqCompaction) {
+    this.maxSelectUnseqFileNumInEachUnseqCompaction = maxSelectUnseqFileNumInEachUnseqCompaction;
+  }
+
   public int getMergeChunkSubThreadNum() {
     return mergeChunkSubThreadNum;
   }
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 8a133da..2632f57 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -342,6 +342,12 @@ public class IoTDBDescriptor {
               properties.getProperty(
                   "unseq_level_num", Integer.toString(conf.getUnseqLevelNum()))));
 
+      conf.setMaxSelectUnseqFileNumInEachUnseqCompaction(
+          Integer.parseInt(
+              properties.getProperty(
+                  "max_open_file_num_in_each_unseq_compaction",
+                  Integer.toString(conf.getMaxSelectUnseqFileNumInEachUnseqCompaction()))));
+
       conf.setUnseqFileNumInEachLevel(
           Integer.parseInt(
               properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index a5ea1da..ce813ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -76,6 +76,8 @@ public abstract class TsFileManagement {
   protected boolean isMergeExecutedInCurrentTask = false;
 
   protected boolean isForceFullMerge = IoTDBDescriptor.getInstance().getConfig().isForceFullMerge();
+  private final int maxOpenFileNumInEachUnseqCompaction =
+      IoTDBDescriptor.getInstance().getConfig().getMaxSelectUnseqFileNumInEachUnseqCompaction();
 
   public TsFileManagement(String storageGroupName, String storageGroupDir) {
     this.storageGroupName = storageGroupName;
@@ -230,6 +232,16 @@ public abstract class TsFileManagement {
       return;
     }
 
+    if (unSeqMergeList.size() > maxOpenFileNumInEachUnseqCompaction) {
+      logger.info(
+          "{} too much unseq files to be merged, reduce it to {}",
+          storageGroupName,
+          maxOpenFileNumInEachUnseqCompaction);
+      unSeqMergeList =
+          unSeqMergeList.subList(
+              unSeqMergeList.size() - maxOpenFileNumInEachUnseqCompaction, unSeqMergeList.size());
+    }
+
     long budget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget();
     long timeLowerBound = System.currentTimeMillis() - dataTTL;
     MergeResource mergeResource = new MergeResource(seqMergeList, unSeqMergeList, timeLowerBound);