You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 03:54:02 UTC
[incubator-iotdb] 01/01: make dynamic controlable
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch make_dynamic_controlable
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fa236efaf3b2cd0e2bde574281ae58765f262ed8
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 11:47:17 2020 +0800
make dynamic controlable
---
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 9 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 98 +++++++++++++---------
4 files changed, 71 insertions(+), 45 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 2069043..58f637a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -127,6 +127,9 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public synchronized boolean tryToAdaptParameters() {
+ if(!CONFIG.isEnableParameterAdapter()){
+ return true;
+ }
boolean canAdjust = true;
double ratio = CompressionRatio.getInstance().getRatio();
long memtableSizeInByte = calcMemTableSize(ratio);
@@ -216,12 +219,12 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException {
- totalStorageGroup += diff;
- maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
if(!CONFIG.isEnableParameterAdapter()){
- CONFIG.setMaxMemtableNumber(maxMemTableNum);
return;
}
+ totalStorageGroup += diff;
+ maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
+ CONFIG.setMaxMemtableNumber(maxMemTableNum);
if (!tryToAdaptParameters()) {
totalStorageGroup -= diff;
maxMemTableNum -= IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index 1703844..78cdeda 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager;
import org.apache.iotdb.db.engine.memtable.IMemTable;
@@ -86,7 +87,9 @@ public class MemTableFlushTask {
}
encodingTaskQueue.add(new EndChunkGroupIoTask());
}
- ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup);
+ }
noMoreEncodingTask = true;
logger.debug(
"Storage group {} memtable {}, flushing into disk: data sort time cost {} ms.",
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 7e554cf..7d7dafe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -261,6 +261,9 @@ public class TsFileProcessor {
*/
private long getMemtableSizeThresholdBasedOnSeriesNum() {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ if(!config.isEnableParameterAdapter()){
+ return config.getMemtableSizeThreshold();
+ }
long memTableSize = (long) (config.getMemtableSizeThreshold() * config.getMaxMemtableNumber()
/ IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup()
* ActiveTimeSeriesCounter.getInstance()
@@ -268,7 +271,6 @@ public class TsFileProcessor {
return Math.max(memTableSize, config.getMemtableSizeThreshold());
}
-
public boolean shouldClose() {
long fileSize = tsFileResource.getFileSize();
long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig()
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0ae8fbf..8f578c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -136,20 +136,22 @@ public class MManager {
try {
initFromLog(logFile);
- // storage group name -> the series number
- seriesNumberInStorageGroups = new HashMap<>();
- List<String> storageGroups = mtree.getAllStorageGroupNames();
- for (String sg : storageGroups) {
- MNode node = mtree.getNodeByPath(sg);
- seriesNumberInStorageGroups.put(sg, node.getLeafCount());
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ // storage group name -> the series number
+ seriesNumberInStorageGroups = new HashMap<>();
+ List<String> storageGroups = mtree.getAllStorageGroupNames();
+ for (String sg : storageGroups) {
+ MNode node = mtree.getNodeByPath(sg);
+ seriesNumberInStorageGroups.put(sg, node.getLeafCount());
+ }
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
}
- if (seriesNumberInStorageGroups.isEmpty()) {
- maxSeriesNumberAmongStorageGroup = 0;
- } else {
- maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
- .max(Integer::compareTo).get();
- }
writeToLog = true;
} catch (IOException | MetadataException e) {
mtree = new MTree();
@@ -184,7 +186,9 @@ public class MManager {
try {
this.mtree = new MTree();
this.mNodeCache.clear();
- this.seriesNumberInStorageGroups.clear();
+ if (seriesNumberInStorageGroups != null) {
+ this.seriesNumberInStorageGroups.clear();
+ }
this.maxSeriesNumberAmongStorageGroup = 0;
if (logWriter != null) {
logWriter.close();
@@ -294,10 +298,12 @@ public class MManager {
createTimeseriesWithMemoryCheckAndLog(path, dataType, encoding, compressor, props);
// update statistics
- int size = seriesNumberInStorageGroups.get(storageGroupName);
- seriesNumberInStorageGroups.put(storageGroupName, size + 1);
- if (size + 1 > maxSeriesNumberAmongStorageGroup) {
- maxSeriesNumberAmongStorageGroup = size + 1;
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ int size = seriesNumberInStorageGroups.get(storageGroupName);
+ seriesNumberInStorageGroups.put(storageGroupName, size + 1);
+ if (size + 1 > maxSeriesNumberAmongStorageGroup) {
+ maxSeriesNumberAmongStorageGroup = size + 1;
+ }
}
} finally {
lock.writeLock().unlock();
@@ -361,12 +367,16 @@ public class MManager {
public Set<String> deleteTimeseries(String prefixPath) throws MetadataException {
lock.writeLock().lock();
if (isStorageGroup(prefixPath)) {
- int size = seriesNumberInStorageGroups.get(prefixPath);
- seriesNumberInStorageGroups.put(prefixPath, 0);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ int size = seriesNumberInStorageGroups.get(prefixPath);
+ seriesNumberInStorageGroups.put(prefixPath, 0);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
+ .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+ }
}
+
mNodeCache.clear();
}
try {
@@ -412,12 +422,15 @@ public class MManager {
} catch (ConfigAdjusterException e) {
throw new MetadataException(e);
}
- String storageGroup = getStorageGroupName(path);
- int size = seriesNumberInStorageGroups.get(storageGroup);
- seriesNumberInStorageGroups.put(storageGroup, size - 1);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ String storageGroup = getStorageGroupName(path);
+ int size = seriesNumberInStorageGroups.get(storageGroup);
+ seriesNumberInStorageGroups.put(storageGroup, size - 1);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
+ .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+ }
}
return storageGroupName;
} finally {
@@ -441,7 +454,9 @@ public class MManager {
writer.flush();
}
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
- ActiveTimeSeriesCounter.getInstance().init(storageGroup);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ ActiveTimeSeriesCounter.getInstance().init(storageGroup);
+ }
seriesNumberInStorageGroups.put(storageGroup, 0);
} catch (IOException e) {
throw new MetadataException(e.getMessage());
@@ -473,17 +488,20 @@ public class MManager {
writer.flush();
}
mNodeCache.clear();
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1);
- int size = seriesNumberInStorageGroups.get(storageGroup);
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1);
- ActiveTimeSeriesCounter.getInstance().delete(storageGroup);
- seriesNumberInStorageGroups.remove(storageGroup);
- if (size == maxSeriesNumberAmongStorageGroup) {
- if (seriesNumberInStorageGroups.isEmpty()) {
- maxSeriesNumberAmongStorageGroup = 0;
- } else {
- maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
- .max(Integer::compareTo).get();
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1);
+ int size = seriesNumberInStorageGroups.get(storageGroup);
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1);
+ ActiveTimeSeriesCounter.getInstance().delete(storageGroup);
+ seriesNumberInStorageGroups.remove(storageGroup);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
}
}
}