You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/10/23 07:44:39 UTC

[incubator-iotdb] branch limit_chunk_threshold created (now 70382fc)

This is an automated email from the ASF dual-hosted git repository.

lta pushed a change to branch limit_chunk_threshold
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.


      at 70382fc  modify the cal equation of memtable threshold

This branch includes the following new commits:

     new 70382fc  modify the cal equation of memtable threshold

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[incubator-iotdb] 01/01: modify the cal equation of memtable threshold

Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lta pushed a commit to branch limit_chunk_threshold
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git

commit 70382fc61f8f3a8497c0d8c275645d1ca26b69f2
Author: lta <li...@163.com>
AuthorDate: Wed Oct 23 15:44:11 2019 +0800

    modify the cal equation of memtable threshold
---
 .../org/apache/iotdb/db/conf/IoTDBConstant.java    |  2 ++
 .../db/conf/adapter/IoTDBConfigDynamicAdapter.java |  5 +--
 .../db/engine/storagegroup/TsFileProcessor.java    | 36 ++++++++++++++++++++--
 .../org/apache/iotdb/db/metadata/MManager.java     | 20 ++++++++++--
 4 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 568b7aa..caf7e10 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -65,4 +65,6 @@ public class IoTDBConstant {
   public static final String USER = "User";
   public static final String PRIVILEGE = "Privilege";
 
+  public static final int MEMTABLE_NUM_INCREMETN = 4;
+
 }
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index f3a71f1..cf72cc3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -19,6 +19,7 @@
 package org.apache.iotdb.db.conf.adapter;
 
 import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.exception.ConfigAdjusterException;
 import org.apache.iotdb.db.metadata.MManager;
@@ -211,14 +212,14 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
   @Override
   public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException {
     totalStorageGroup += diff;
-    maxMemTableNum += 4 * diff;
+    maxMemTableNum += IoTDBConstant.MEMTABLE_NUM_INCREMETN * diff;
     if(!CONFIG.isEnableParameterAdapter()){
       CONFIG.setMaxMemtableNumber(maxMemTableNum);
       return;
     }
     if (!tryToAdaptParameters()) {
       totalStorageGroup -= diff;
-      maxMemTableNum -= 4 * diff;
+      maxMemTableNum -= IoTDBConstant.MEMTABLE_NUM_INCREMETN * diff;
       throw new ConfigAdjusterException(
           "The IoTDB system load is too large to create storage group.");
     }
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 902f14f..5a1d3f4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -29,9 +29,11 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Supplier;
+import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.conf.adapter.CompressionRatio;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
 import org.apache.iotdb.db.engine.flush.FlushManager;
 import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
 import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
@@ -45,15 +47,16 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.CloseTsFile
 import org.apache.iotdb.db.engine.version.VersionController;
 import org.apache.iotdb.db.exception.TsFileProcessorException;
 import org.apache.iotdb.db.exception.qp.QueryProcessorException;
+import org.apache.iotdb.db.metadata.MManager;
 import org.apache.iotdb.db.qp.constant.DatetimeUtils;
 import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
 import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
 import org.apache.iotdb.db.query.context.QueryContext;
 import org.apache.iotdb.db.rescon.MemTablePool;
-import org.apache.iotdb.rpc.TSStatusCode;
 import org.apache.iotdb.db.utils.QueryUtils;
 import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
 import org.apache.iotdb.db.writelog.node.WriteLogNode;
+import org.apache.iotdb.rpc.TSStatusCode;
 import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
 import org.apache.iotdb.tsfile.utils.Pair;
@@ -223,8 +226,35 @@ public class TsFileProcessor {
 
 
   boolean shouldFlush() {
-    return workMemTable != null && workMemTable.memSize() > IoTDBDescriptor.getInstance()
-        .getConfig().getMemtableSizeThreshold();
+    return workMemTable != null
+        && workMemTable.memSize() > getMemtableSizeThresholdBasedOnSeriesNum();
+  }
+
+  /**
+   * <p>In the dynamic parameter adjustment module{@link IoTDBConfigDynamicAdapter}, it calculated
+   * the average size of each metatable{@link IoTDBConfigDynamicAdapter#tryToAdaptParameters()}.
+   * However, considering that the number of timeseries between storage groups may vary greatly,
+   * it's appropriate to judge whether to flush the memtable according to the average memtable size.
+   * We need to adjust it according to the number of timeseries in a specific storage group.
+   *
+   * Abbreviation of parameters:
+   *
+   * 1 memtableSize: m
+   * 2 maxMemTableNum: Nm
+   * 3 SeriesNumber: Ns
+   * 4 chunkSizeThreshold: Sc
+   * 5 Total timeseries number: Nts  {@link IoTDBConfigDynamicAdapter#totalTimeseries}
+   * 6 MemTable Number for Each SG: Nmes  {@link IoTDBConstant#MEMTABLE_NUM_INCREMETN}
+   *
+   * <p>The equation: Σ(Ns * Sc) * Nmes = m * Nm  ==> Σ(Ns) * Sc * Nmes = m * Nm ==> Sc = m * Nm / Nmes / Nts
+   * <p>Note: Σ means the sum of storage groups , so Nts = ΣNs
+   *
+   */
+  private long getMemtableSizeThresholdBasedOnSeriesNum() {
+    IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+    return config.getMemtableSizeThreshold() * config.getMaxMemtableNumber()
+        / IoTDBConstant.MEMTABLE_NUM_INCREMETN / IoTDBConfigDynamicAdapter.getInstance()
+        .getTotalTimeseries() * MManager.getInstance().getSeriesNumber(storageGroupName);
   }
 
 
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8079417..f10d8c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -18,10 +18,20 @@
  */
 package org.apache.iotdb.db.metadata;
 
-import java.io.*;
-import java.util.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -1279,6 +1289,10 @@ public class MManager {
     }
   }
 
+  public int getSeriesNumber(String storageGroup) {
+    return seriesNumberInStorageGroups.getOrDefault(storageGroup, 0);
+  }
+
   /**
    * function for getting storage group name when creating schema automatically is enable
    */