You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/07/20 08:51:27 UTC
[incubator-iotdb] branch improve_calmem_tool created (now 1339218)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch improve_calmem_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 1339218 accelerate cal mem
This branch includes the following new commits:
new 1339218 accelerate cal mem
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: accelerate cal mem
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch improve_calmem_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 1339218291918c9201198cc0938db35487dbe25f
Author: lta <li...@163.com>
AuthorDate: Sat Jul 20 16:51:10 2019 +0800
accelerate cal mem
---
.../iotdb/db/tools/MemEst/MemEstToolCmd.java | 36 ++++++++++++++++------
1 file changed, 27 insertions(+), 9 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
index 9f7977b..75cedf5 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.db.tools.MemEst;
import io.airlift.airline.Command;
import io.airlift.airline.Option;
-import io.airlift.airline.OptionType;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -45,20 +44,27 @@ public class MemEstToolCmd implements Runnable {
@Override
public void run() {
+ // backup origin config parameters
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
long memTableSize = config.getMemtableSizeThreshold();
int maxMemtableNumber = config.getMaxMemtableNumber();
long tsFileSize = config.getTsFileSizeThreshold();
- long memory = IoTDBConstant.GB;
+
+ // parse input parameters
long sgNum = Long.parseLong(sgNumString);
long tsNum = Long.parseLong(tsNumString);
long maxTsNum = Long.parseLong(maxTsNumString);
+
+ //
+ long stepMemory = calStepMemory(tsNum) * IoTDBConstant.GB;
+ long currentMemory = stepMemory;
long maxTsNumValid = maxTsNum;
long maxProcess = 0;
-
+ long start = System.currentTimeMillis();
while (true) {
- // init parameter
- config.setAllocateMemoryForWrite(memory);
+
+ // recover config parameter
+ config.setAllocateMemoryForWrite(currentMemory);
config.setMemtableSizeThreshold(memTableSize);
config.setMaxMemtableNumber(maxMemtableNumber);
config.setTsFileSizeThreshold(tsFileSize);
@@ -74,7 +80,7 @@ public class MemEstToolCmd implements Runnable {
}
for (; tsCnt <= tsNum; tsCnt++) {
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
- if(maxTsNum == 0){
+ if (maxTsNum == 0) {
maxTsNumValid = tsCnt / sgNum + 1;
} else {
maxTsNumValid = Math.min(tsCnt, maxTsNum);
@@ -84,18 +90,30 @@ public class MemEstToolCmd implements Runnable {
}
} catch (ConfigAdjusterException e) {
- if(sgCnt > sgNum) {
+ if (sgCnt > sgNum) {
maxProcess = Math.max(maxProcess, tsCnt * 100 / tsNum);
System.out
.print(String.format("Memory estimation progress : %d%%\r", maxProcess));
}
- memory += IoTDBConstant.GB;
+ currentMemory += stepMemory;
continue;
}
break;
}
System.out.println(String
- .format("Memory for writing: %dGB, SG: %d, TS: %d, MTS: %d", memory / IoTDBConstant.GB,
+ .format("Memory for writing: %dGB, SG: %d, TS: %d, MTS: %d", currentMemory / IoTDBConstant.GB,
sgNum, tsNum, maxTsNumValid));
+ System.out.println(String.format("Calculating memory for writing consumes: %dms",
+ (System.currentTimeMillis() - start)));
+ }
+
+ private long calStepMemory(long maxTimeseriesNumber) {
+ maxTimeseriesNumber /= 10000000;
+ int step = 1;
+ while (maxTimeseriesNumber > 0) {
+ maxTimeseriesNumber /= 10;
+ step *= 10;
+ }
+ return step;
}
}