You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 03:54:01 UTC
[incubator-iotdb] branch make_dynamic_controlable created (now
fa236ef)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch make_dynamic_controlable
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at fa236ef make dynamic controlable
This branch includes the following new commits:
new fa236ef make dynamic controlable
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: make dynamic controlable
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch make_dynamic_controlable
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fa236efaf3b2cd0e2bde574281ae58765f262ed8
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 11:47:17 2020 +0800
make dynamic controlable
---
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 9 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 98 +++++++++++++---------
4 files changed, 71 insertions(+), 45 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 2069043..58f637a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -127,6 +127,9 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public synchronized boolean tryToAdaptParameters() {
+ if(!CONFIG.isEnableParameterAdapter()){
+ return true;
+ }
boolean canAdjust = true;
double ratio = CompressionRatio.getInstance().getRatio();
long memtableSizeInByte = calcMemTableSize(ratio);
@@ -216,12 +219,12 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException {
- totalStorageGroup += diff;
- maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
if(!CONFIG.isEnableParameterAdapter()){
- CONFIG.setMaxMemtableNumber(maxMemTableNum);
return;
}
+ totalStorageGroup += diff;
+ maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
+ CONFIG.setMaxMemtableNumber(maxMemTableNum);
if (!tryToAdaptParameters()) {
totalStorageGroup -= diff;
maxMemTableNum -= IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index 1703844..78cdeda 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager;
import org.apache.iotdb.db.engine.memtable.IMemTable;
@@ -86,7 +87,9 @@ public class MemTableFlushTask {
}
encodingTaskQueue.add(new EndChunkGroupIoTask());
}
- ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup);
+ }
noMoreEncodingTask = true;
logger.debug(
"Storage group {} memtable {}, flushing into disk: data sort time cost {} ms.",
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 7e554cf..7d7dafe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -261,6 +261,9 @@ public class TsFileProcessor {
*/
private long getMemtableSizeThresholdBasedOnSeriesNum() {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ if(!config.isEnableParameterAdapter()){
+ return config.getMemtableSizeThreshold();
+ }
long memTableSize = (long) (config.getMemtableSizeThreshold() * config.getMaxMemtableNumber()
/ IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup()
* ActiveTimeSeriesCounter.getInstance()
@@ -268,7 +271,6 @@ public class TsFileProcessor {
return Math.max(memTableSize, config.getMemtableSizeThreshold());
}
-
public boolean shouldClose() {
long fileSize = tsFileResource.getFileSize();
long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig()
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0ae8fbf..8f578c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -136,20 +136,22 @@ public class MManager {
try {
initFromLog(logFile);
- // storage group name -> the series number
- seriesNumberInStorageGroups = new HashMap<>();
- List<String> storageGroups = mtree.getAllStorageGroupNames();
- for (String sg : storageGroups) {
- MNode node = mtree.getNodeByPath(sg);
- seriesNumberInStorageGroups.put(sg, node.getLeafCount());
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ // storage group name -> the series number
+ seriesNumberInStorageGroups = new HashMap<>();
+ List<String> storageGroups = mtree.getAllStorageGroupNames();
+ for (String sg : storageGroups) {
+ MNode node = mtree.getNodeByPath(sg);
+ seriesNumberInStorageGroups.put(sg, node.getLeafCount());
+ }
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
}
- if (seriesNumberInStorageGroups.isEmpty()) {
- maxSeriesNumberAmongStorageGroup = 0;
- } else {
- maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
- .max(Integer::compareTo).get();
- }
writeToLog = true;
} catch (IOException | MetadataException e) {
mtree = new MTree();
@@ -184,7 +186,9 @@ public class MManager {
try {
this.mtree = new MTree();
this.mNodeCache.clear();
- this.seriesNumberInStorageGroups.clear();
+ if (seriesNumberInStorageGroups != null) {
+ this.seriesNumberInStorageGroups.clear();
+ }
this.maxSeriesNumberAmongStorageGroup = 0;
if (logWriter != null) {
logWriter.close();
@@ -294,10 +298,12 @@ public class MManager {
createTimeseriesWithMemoryCheckAndLog(path, dataType, encoding, compressor, props);
// update statistics
- int size = seriesNumberInStorageGroups.get(storageGroupName);
- seriesNumberInStorageGroups.put(storageGroupName, size + 1);
- if (size + 1 > maxSeriesNumberAmongStorageGroup) {
- maxSeriesNumberAmongStorageGroup = size + 1;
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ int size = seriesNumberInStorageGroups.get(storageGroupName);
+ seriesNumberInStorageGroups.put(storageGroupName, size + 1);
+ if (size + 1 > maxSeriesNumberAmongStorageGroup) {
+ maxSeriesNumberAmongStorageGroup = size + 1;
+ }
}
} finally {
lock.writeLock().unlock();
@@ -361,12 +367,16 @@ public class MManager {
public Set<String> deleteTimeseries(String prefixPath) throws MetadataException {
lock.writeLock().lock();
if (isStorageGroup(prefixPath)) {
- int size = seriesNumberInStorageGroups.get(prefixPath);
- seriesNumberInStorageGroups.put(prefixPath, 0);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ int size = seriesNumberInStorageGroups.get(prefixPath);
+ seriesNumberInStorageGroups.put(prefixPath, 0);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
+ .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+ }
}
+
mNodeCache.clear();
}
try {
@@ -412,12 +422,15 @@ public class MManager {
} catch (ConfigAdjusterException e) {
throw new MetadataException(e);
}
- String storageGroup = getStorageGroupName(path);
- int size = seriesNumberInStorageGroups.get(storageGroup);
- seriesNumberInStorageGroups.put(storageGroup, size - 1);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ String storageGroup = getStorageGroupName(path);
+ int size = seriesNumberInStorageGroups.get(storageGroup);
+ seriesNumberInStorageGroups.put(storageGroup, size - 1);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
+ .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+ }
}
return storageGroupName;
} finally {
@@ -441,7 +454,9 @@ public class MManager {
writer.flush();
}
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
- ActiveTimeSeriesCounter.getInstance().init(storageGroup);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ ActiveTimeSeriesCounter.getInstance().init(storageGroup);
+ }
seriesNumberInStorageGroups.put(storageGroup, 0);
} catch (IOException e) {
throw new MetadataException(e.getMessage());
@@ -473,17 +488,20 @@ public class MManager {
writer.flush();
}
mNodeCache.clear();
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1);
- int size = seriesNumberInStorageGroups.get(storageGroup);
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1);
- ActiveTimeSeriesCounter.getInstance().delete(storageGroup);
- seriesNumberInStorageGroups.remove(storageGroup);
- if (size == maxSeriesNumberAmongStorageGroup) {
- if (seriesNumberInStorageGroups.isEmpty()) {
- maxSeriesNumberAmongStorageGroup = 0;
- } else {
- maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
- .max(Integer::compareTo).get();
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1);
+ int size = seriesNumberInStorageGroups.get(storageGroup);
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1);
+ ActiveTimeSeriesCounter.getInstance().delete(storageGroup);
+ seriesNumberInStorageGroups.remove(storageGroup);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
}
}
}