You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/10/24 04:48:13 UTC
[incubator-iotdb] branch master updated: [IOTDB-265]Re-adjust the
threshold size of memtable (#471)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 17e6327 [IOTDB-265]Re-adjust the threshold size of memtable (#471)
17e6327 is described below
commit 17e6327b29d2203824138893f64b4d84ab1abe36
Author: Tianan Li <li...@163.com>
AuthorDate: Thu Oct 24 12:48:08 2019 +0800
[IOTDB-265]Re-adjust the threshold size of memtable (#471)
* modify the cal equation of memtable threshold
---
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 2 ++
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 5 +--
.../db/engine/storagegroup/TsFileProcessor.java | 38 ++++++++++++++++++++--
.../org/apache/iotdb/db/metadata/MManager.java | 20 ++++++++++--
4 files changed, 57 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 568b7aa..096452e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -65,4 +65,6 @@ public class IoTDBConstant {
public static final String USER = "User";
public static final String PRIVILEGE = "Privilege";
+ public static final int MEMTABLE_NUM_IN_EACH_STORAGE_GROUP = 4;
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index f3a71f1..bc4f144 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.conf.adapter;
import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.ConfigAdjusterException;
import org.apache.iotdb.db.metadata.MManager;
@@ -211,14 +212,14 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException {
totalStorageGroup += diff;
- maxMemTableNum += 4 * diff;
+ maxMemTableNum += IoTDBConstant.MEMTABLE_NUM_IN_EACH_STORAGE_GROUP * diff;
if(!CONFIG.isEnableParameterAdapter()){
CONFIG.setMaxMemtableNumber(maxMemTableNum);
return;
}
if (!tryToAdaptParameters()) {
totalStorageGroup -= diff;
- maxMemTableNum -= 4 * diff;
+ maxMemTableNum -= IoTDBConstant.MEMTABLE_NUM_IN_EACH_STORAGE_GROUP * diff;
throw new ConfigAdjusterException(
"The IoTDB system load is too large to create storage group.");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 17aec9d..44a1f91 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -29,9 +29,11 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
+import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
@@ -45,15 +47,16 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.CloseTsFile
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.qp.QueryProcessorException;
+import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.rescon.MemTablePool;
-import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -226,8 +229,37 @@ public class TsFileProcessor {
boolean shouldFlush() {
- return workMemTable != null && workMemTable.memSize() > IoTDBDescriptor.getInstance()
- .getConfig().getMemtableSizeThreshold();
+ return workMemTable != null
+ && workMemTable.memSize() > getMemtableSizeThresholdBasedOnSeriesNum();
+ }
+
+ /**
+ * <p>In the dynamic parameter adjustment module{@link IoTDBConfigDynamicAdapter}, it calculated
+ * the average size of each metatable{@link IoTDBConfigDynamicAdapter#tryToAdaptParameters()}.
+ * However, considering that the number of timeseries between storage groups may vary greatly,
+ * it's appropriate to judge whether to flush the memtable according to the average memtable size.
+ * We need to adjust it according to the number of timeseries in a specific storage group.
+ *
+ * Abbreviation of parameters:
+ *
+ * 1 memtableSize: m
+ * 2 maxMemTableNum: Nm
+ * 3 SeriesNumber: Ns
+ * 4 chunkSizeThreshold: Sc
+ * 5 Total timeseries number: Nts {@link IoTDBConfigDynamicAdapter#totalTimeseries}
+ * 6 MemTable Number for Each SG: Nmes {@link IoTDBConstant#MEMTABLE_NUM_IN_EACH_STORAGE_GROUP}
+ *
+ * <p>The equation: Σ(Ns * Sc) * Nmes = m * Nm ==> Σ(Ns) * Sc * Nmes = m * Nm ==> Sc = m * Nm / Nmes / Nts
+ * <p>Note: Σ means the sum of storage groups , so Nts = ΣNs
+ *
+ */
+ private long getMemtableSizeThresholdBasedOnSeriesNum() {
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ return IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries() == 0 ? config
+ .getMemtableSizeThreshold() :
+ config.getMemtableSizeThreshold() * config.getMaxMemtableNumber()
+ / IoTDBConstant.MEMTABLE_NUM_IN_EACH_STORAGE_GROUP / IoTDBConfigDynamicAdapter.getInstance()
+ .getTotalTimeseries() * MManager.getInstance().getSeriesNumber(storageGroupName);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8079417..f10d8c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -18,10 +18,20 @@
*/
package org.apache.iotdb.db.metadata;
-import java.io.*;
-import java.util.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -1279,6 +1289,10 @@ public class MManager {
}
}
+ public int getSeriesNumber(String storageGroup) {
+ return seriesNumberInStorageGroups.getOrDefault(storageGroup, 0);
+ }
+
/**
* function for getting storage group name when creating schema automatically is enable
*/