You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/07/17 05:03:00 UTC
[incubator-iotdb] 02/02: add memory tool shell script
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch dev_memory_estimate_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 0439c0e4f58d4271569e2c1dc8efc23107182a73
Author: lta <li...@163.com>
AuthorDate: Wed Jul 17 13:01:01 2019 +0800
add memory tool shell script
---
iotdb/iotdb/bin/memory-tool.sh | 12 +--
iotdb/iotdb/bin/momory-tool.bat | 11 ++-
iotdb/iotdb/conf/logback-tool.xml | 22 ++++++
.../apache/iotdb/db/tools/MemEst/MemEstTool.java | 82 +++++++++++++++++++
.../iotdb/db/tools/MemEst/MemEstToolCmd.java | 91 ++++++++++++++++++++++
.../java/org/apache/iotdb/db/tools/WalChecker.java | 1 -
.../adapter/IoTDBConfigDynamicAdapterTest.java | 40 ----------
7 files changed, 209 insertions(+), 50 deletions(-)
diff --git a/iotdb/iotdb/bin/memory-tool.sh b/iotdb/iotdb/bin/memory-tool.sh
index c28fff6..5c2ad0d 100755
--- a/iotdb/iotdb/bin/memory-tool.sh
+++ b/iotdb/iotdb/bin/memory-tool.sh
@@ -22,15 +22,15 @@ if [ -z "${IOTDB_HOME}" ]; then
export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
fi
-
-MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
-
+IOTDB_CONF=${IOTDB_HOME}/conf
+# IOTDB_LOGS=${IOTDB_HOME}/logs
CLASSPATH=""
-for f in ${IOTDB_HOME}/lib_cluster/*.jar; do
+for f in ${IOTDB_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
+MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
if [ -n "$JAVA_HOME" ]; then
for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
@@ -43,6 +43,6 @@ else
JAVA=java
fi
-exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback-tool.xml"
-exit $?
+exec "$JAVA" $iotdb_parms -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
\ No newline at end of file
diff --git a/iotdb/iotdb/bin/momory-tool.bat b/iotdb/iotdb/bin/momory-tool.bat
index 2387a9e..7004ac5 100755
--- a/iotdb/iotdb/bin/momory-tool.bat
+++ b/iotdb/iotdb/bin/momory-tool.bat
@@ -23,7 +23,7 @@ pushd %~dp0..
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
@@ -32,9 +32,14 @@ set JAVA_OPTS=-ea^
-DIOTDB_HOME=%IOTDB_HOME%
REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_HOME%\lib_cluster\*.jar") do call :append "%%i"
+for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
goto okClasspath
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%IOTDB_CONF%\logback-tool.xml"
+
:append
set CLASSPATH=%CLASSPATH%;%1
goto :eof
@@ -42,7 +47,7 @@ goto :eof
REM -----------------------------------------------------------------------------
:okClasspath
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
goto finally
diff --git a/iotdb/iotdb/conf/logback-tool.xml b/iotdb/iotdb/conf/logback-tool.xml
new file mode 100644
index 0000000..98f3f9d
--- /dev/null
+++ b/iotdb/iotdb/conf/logback-tool.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration/>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
new file mode 100644
index 0000000..165131c
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import io.airlift.airline.Cli;
+import io.airlift.airline.Help;
+import io.airlift.airline.ParseArgumentsMissingException;
+import io.airlift.airline.ParseArgumentsUnexpectedException;
+import io.airlift.airline.ParseCommandMissingException;
+import io.airlift.airline.ParseCommandUnrecognizedException;
+import io.airlift.airline.ParseOptionConversionException;
+import io.airlift.airline.ParseOptionMissingException;
+import io.airlift.airline.ParseOptionMissingValueException;
+import java.util.List;
+
+public class MemEstTool {
+
+ public static void main(String... args) {
+ List<Class<? extends Runnable>> commands = Lists.newArrayList(
+ Help.class,
+ MemEstToolCmd.class
+ );
+ Cli.CliBuilder<Runnable> builder = Cli.builder("memory-tool");
+
+ builder.withDescription("Estimate memory for writing")
+ .withDefaultCommand(Help.class)
+ .withCommands(commands);
+
+ Cli<Runnable> parser = builder.build();
+
+ int status = 0;
+ try {
+ Runnable parse = parser.parse(args);
+ parse.run();
+ } catch (IllegalArgumentException |
+ IllegalStateException |
+ ParseArgumentsMissingException |
+ ParseArgumentsUnexpectedException |
+ ParseOptionConversionException |
+ ParseOptionMissingException |
+ ParseOptionMissingValueException |
+ ParseCommandMissingException |
+ ParseCommandUnrecognizedException e) {
+ badUse(e);
+ status = 1;
+ } catch (Exception e) {
+ err(Throwables.getRootCause(e));
+ status = 2;
+ }
+
+ System.exit(status);
+ }
+
+ private static void badUse(Exception e) {
+ System.out.println("memory-tool: " + e.getMessage());
+ System.out.println("See 'memory-tool help' or 'memory-tool help <command>'.");
+ }
+
+ private static void err(Throwable e) {
+ System.err.println("error: " + e.getMessage());
+ System.err.println("-- StackTrace --");
+ System.err.println(Throwables.getStackTraceAsString(e));
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
new file mode 100644
index 0000000..301eefd
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import io.airlift.airline.OptionType;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.exception.ConfigAdjusterException;
+import org.apache.iotdb.db.metadata.MManager;
+
+@Command(name = "memory", description = "calculate minimum memory required for writing based on the number of storage groups and timeseries")
+public class MemEstToolCmd implements Runnable {
+
+ @Option(type = OptionType.GLOBAL, title = "storage group number", name = {"-sg",
+ "--storagegroup"}, description = "Storage group number")
+ private String sgNumString = "10";
+
+ @Option(type = OptionType.GLOBAL, title = "total timeseries number", name = {"-ts",
+ "--tsNum"}, description = "Total timeseries number")
+ private String tsNumString = "1000";
+
+ @Option(title = "max timeseries", name = {"-mts",
+ "--mtsNum"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number")
+ private String maxTsNumString = "0";
+
+ @Override
+ public void run() {
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ long memTableSize = config.getMemtableSizeThreshold();
+ int maxMemtableNumber = config.getMaxMemtableNumber();
+ long tsFileSize = config.getTsFileSizeThreshold();
+ long memory = IoTDBConstant.GB;
+ int sgNum = Integer.parseInt(sgNumString);
+ int tsNum = Integer.parseInt(tsNumString);
+ int maxTsNum = Integer.parseInt(maxTsNumString);
+ while (true) {
+ // init parameter
+ config.setAllocateMemoryForWrite(memory);
+ config.setMemtableSizeThreshold(memTableSize);
+ config.setMaxMemtableNumber(maxMemtableNumber);
+ config.setTsFileSizeThreshold(tsFileSize);
+ IoTDBConfigDynamicAdapter.getInstance().reset();
+ IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
+ MManager.getInstance().clear();
+
+ int sgCnt = 1;
+ int tsCnt = 1;
+ try {
+ for (; sgCnt <= sgNum; sgCnt++) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
+ }
+ for (; tsCnt <= tsNum; tsCnt++) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
+ MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(
+ maxTsNum == 0 ? tsCnt / sgNum + 1 : Math.min(tsCnt, maxTsNum));
+ }
+
+ } catch (ConfigAdjusterException e) {
+ if(sgCnt > sgNum) {
+ System.out
+ .print(String.format("Memory estimation progress : %d%%\r", tsCnt * 100 / tsNum));
+ }
+ memory += IoTDBConstant.GB;
+ continue;
+ }
+ break;
+ }
+ System.out.println(String.format("SG: %d, TS: %d, MTS: %d, Memory for writing: %dGB", sgNum,
+ tsNum, maxTsNum == 0 ? tsNum / sgNum + 1 : maxTsNum, memory / IoTDBConstant.GB));
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
index d06df82..d390403 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iotdb.db.tools;
import static org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode.WAL_FILE_NAME;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
index 27fe9e3..1222935 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
@@ -146,44 +146,4 @@ public class IoTDBConfigDynamicAdapterTest {
assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
}
}
-
- @Test
- public void addOrDeleteTimeSeriesSyso2() {
- int sgNum = 1;
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- long memTableSize = config.getMemtableSizeThreshold();
- int maxMemtableNumber = config.getMaxMemtableNumber();
- long tsFileSize = config.getTsFileSizeThreshold();
- long memory = 1024 * 1024 * 1024L;
- while (true) {
- config.setAllocateMemoryForWrite(memory);
- config.setMemtableSizeThreshold(memTableSize);
- config.setMaxMemtableNumber(maxMemtableNumber);
- config.setTsFileSizeThreshold(tsFileSize);
- IoTDBConfigDynamicAdapter.getInstance().reset();
- IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
- MManager.getInstance().clear();
- for (int i = 1; i <= 50 ; i++) {
- try {
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum);
- } catch (ConfigAdjusterException e) {
- e.printStackTrace();
- }
- }
- int i = 1;
- try {
- for (; i <= 10000000; i++) {
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
- MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 50 + 1);
- }
- } catch (ConfigAdjusterException e) {
-// System.out.println(i);
- memory += 1024 * 1024 * 1024L;
-// System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
- continue;
- }
- break;
- }
- System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
- }
}
\ No newline at end of file