You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/07/17 06:15:21 UTC
[incubator-iotdb] branch dev_memory_estimate_tool updated (0439c0e
-> 0314285)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch dev_memory_estimate_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
discard 0439c0e add memory tool shell script
new 0314285 add memory tool shell script
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0439c0e)
\
N -- N -- N refs/heads/dev_memory_estimate_tool (0314285)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
iotdb/iotdb/bin/{momory-tool.bat => memory-tool.bat} | 6 ++++--
iotdb/iotdb/bin/start-server.bat | 4 ++--
iotdb/iotdb/bin/start-sync-client.bat | 6 +++---
.../org/apache/iotdb/db/conf/adapter/CompressionRatio.java | 12 +++++-------
.../java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java | 8 ++++++--
5 files changed, 20 insertions(+), 16 deletions(-)
rename iotdb/iotdb/bin/{momory-tool.bat => memory-tool.bat} (90%)
[incubator-iotdb] 01/01: add memory tool shell script
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch dev_memory_estimate_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 03142854599fc3682928886d2d9442de5578ac21
Author: lta <li...@163.com>
AuthorDate: Wed Jul 17 13:01:01 2019 +0800
add memory tool shell script
---
.../iotdb/bin/{momory-tool.bat => memory-tool.bat} | 17 ++--
iotdb/iotdb/bin/memory-tool.sh | 12 +--
iotdb/iotdb/bin/start-server.bat | 4 +-
iotdb/iotdb/bin/start-sync-client.bat | 6 +-
iotdb/iotdb/conf/logback-tool.xml | 22 ++++++
.../iotdb/db/conf/adapter/CompressionRatio.java | 12 ++-
.../apache/iotdb/db/tools/MemEst/MemEstTool.java | 86 ++++++++++++++++++++
.../iotdb/db/tools/MemEst/MemEstToolCmd.java | 91 ++++++++++++++++++++++
.../java/org/apache/iotdb/db/tools/WalChecker.java | 1 -
.../adapter/IoTDBConfigDynamicAdapterTest.java | 40 ----------
10 files changed, 227 insertions(+), 64 deletions(-)
diff --git a/iotdb/iotdb/bin/momory-tool.bat b/iotdb/iotdb/bin/memory-tool.bat
similarity index 68%
rename from iotdb/iotdb/bin/momory-tool.bat
rename to iotdb/iotdb/bin/memory-tool.bat
index 2387a9e..3d652f3 100755
--- a/iotdb/iotdb/bin/momory-tool.bat
+++ b/iotdb/iotdb/bin/memory-tool.bat
@@ -17,13 +17,15 @@
@REM under the License.
@REM
+
+@echo off
if "%OS%" == "Windows_NT" setlocal
pushd %~dp0..
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
@@ -31,18 +33,23 @@ if NOT DEFINED JAVA_HOME goto :err
set JAVA_OPTS=-ea^
-DIOTDB_HOME=%IOTDB_HOME%
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_HOME%\lib_cluster\*.jar") do call :append "%%i"
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
goto okClasspath
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%IOTDB_CONF%\logback-tool.xml"
+
:append
set CLASSPATH=%CLASSPATH%;%1
goto :eof
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
:okClasspath
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
goto finally
diff --git a/iotdb/iotdb/bin/memory-tool.sh b/iotdb/iotdb/bin/memory-tool.sh
index c28fff6..5c2ad0d 100755
--- a/iotdb/iotdb/bin/memory-tool.sh
+++ b/iotdb/iotdb/bin/memory-tool.sh
@@ -22,15 +22,15 @@ if [ -z "${IOTDB_HOME}" ]; then
export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
fi
-
-MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
-
+IOTDB_CONF=${IOTDB_HOME}/conf
+# IOTDB_LOGS=${IOTDB_HOME}/logs
CLASSPATH=""
-for f in ${IOTDB_HOME}/lib_cluster/*.jar; do
+for f in ${IOTDB_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
+MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
if [ -n "$JAVA_HOME" ]; then
for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
@@ -43,6 +43,6 @@ else
JAVA=java
fi
-exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback-tool.xml"
-exit $?
+exec "$JAVA" $iotdb_parms -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
\ No newline at end of file
diff --git a/iotdb/iotdb/bin/start-server.bat b/iotdb/iotdb/bin/start-server.bat
index 2077797..2bbabd5 100755
--- a/iotdb/iotdb/bin/start-server.bat
+++ b/iotdb/iotdb/bin/start-server.bat
@@ -78,7 +78,7 @@ set JAVA_OPTS=-ea^
@REM Ensure that any user defined CLASSPATH variables are not used on startup
set CLASSPATH="%IOTDB_HOME%\lib"
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
set CLASSPATH=%CLASSPATH%;iotdb.IoTDB
goto okClasspath
@@ -87,7 +87,7 @@ goto okClasspath
set CLASSPATH=%CLASSPATH%;%1
goto :eof
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
:okClasspath
rem echo CLASSPATH: %CLASSPATH%
diff --git a/iotdb/iotdb/bin/start-sync-client.bat b/iotdb/iotdb/bin/start-sync-client.bat
index 219f939..49360ef 100755
--- a/iotdb/iotdb/bin/start-sync-client.bat
+++ b/iotdb/iotdb/bin/start-sync-client.bat
@@ -44,7 +44,7 @@ set JAVA_OPTS=-ea^
@REM Ensure that any user defined CLASSPATH variables are not used on startup
set CLASSPATH="%IOTDB_HOME%\lib"
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
set CLASSPATH=%CLASSPATH%;SyncClient
goto okClasspath
@@ -53,10 +53,10 @@ goto okClasspath
set CLASSPATH=%CLASSPATH%;%1
goto :eof
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
:okClasspath
-rem echo CLASSPATH: %CLASSPATH%
+@rem echo CLASSPATH: %CLASSPATH%
"%JAVA_HOME%\bin\java" %JAVA_OPTS% %IOTDB_HEAP_OPTS% -cp %CLASSPATH% %IOTDB_JMX_OPTS% %MAIN_CLASS%
goto finally
diff --git a/iotdb/iotdb/conf/logback-tool.xml b/iotdb/iotdb/conf/logback-tool.xml
new file mode 100644
index 0000000..98f3f9d
--- /dev/null
+++ b/iotdb/iotdb/conf/logback-tool.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration/>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
index 65afce8..1452c1a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
@@ -68,11 +68,7 @@ public class CompressionRatio {
private CompressionRatio() {
directory = new File(
FilePathUtils.regularizePath(CONFIG.getSystemDir()) + COMPRESSION_RATIO_DIR);
- try {
- restore();
- } catch (IOException e) {
- LOGGER.error("Can not restore CompressionRatio", e);
- }
+ restore();
}
/**
@@ -123,8 +119,10 @@ public class CompressionRatio {
/**
* Restore compression ratio statistics from disk when system restart
*/
- void restore() throws IOException {
- checkDirectoryExist();
+ void restore() {
+ if (!directory.exists()) {
+ return;
+ }
File[] ratioFiles = directory.listFiles((dir, name) -> name.startsWith(FILE_PREFIX));
if (ratioFiles != null && ratioFiles.length > 0) {
long maxTimes = 0;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
new file mode 100644
index 0000000..3f974b1
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import io.airlift.airline.Cli;
+import io.airlift.airline.Help;
+import io.airlift.airline.ParseArgumentsMissingException;
+import io.airlift.airline.ParseArgumentsUnexpectedException;
+import io.airlift.airline.ParseCommandMissingException;
+import io.airlift.airline.ParseCommandUnrecognizedException;
+import io.airlift.airline.ParseOptionConversionException;
+import io.airlift.airline.ParseOptionMissingException;
+import io.airlift.airline.ParseOptionMissingValueException;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+public class MemEstTool {
+
+ public static void main(String... args) throws IOException {
+ List<Class<? extends Runnable>> commands = Lists.newArrayList(
+ Help.class,
+ MemEstToolCmd.class
+ );
+ Cli.CliBuilder<Runnable> builder = Cli.builder("memory-tool");
+
+ builder.withDescription("Estimate memory for writing")
+ .withDefaultCommand(Help.class)
+ .withCommands(commands);
+
+ Cli<Runnable> parser = builder.build();
+
+ int status = 0;
+ try {
+ Runnable parse = parser.parse(args);
+ parse.run();
+ } catch (IllegalArgumentException |
+ IllegalStateException |
+ ParseArgumentsMissingException |
+ ParseArgumentsUnexpectedException |
+ ParseOptionConversionException |
+ ParseOptionMissingException |
+ ParseOptionMissingValueException |
+ ParseCommandMissingException |
+ ParseCommandUnrecognizedException e) {
+ badUse(e);
+ status = 1;
+ } catch (Exception e) {
+ err(Throwables.getRootCause(e));
+ status = 2;
+ }
+ FileUtils.deleteDirectory(new File(IoTDBDescriptor.getInstance().getConfig().getBaseDir()));
+ System.exit(status);
+ }
+
+ private static void badUse(Exception e) {
+ System.out.println("memory-tool: " + e.getMessage());
+ System.out.println("See 'memory-tool help' or 'memory-tool help <command>'.");
+ }
+
+ private static void err(Throwable e) {
+ System.err.println("error: " + e.getMessage());
+ System.err.println("-- StackTrace --");
+ System.err.println(Throwables.getStackTraceAsString(e));
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
new file mode 100644
index 0000000..301eefd
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import io.airlift.airline.OptionType;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.exception.ConfigAdjusterException;
+import org.apache.iotdb.db.metadata.MManager;
+
+@Command(name = "memory", description = "calculate minimum memory required for writing based on the number of storage groups and timeseries")
+public class MemEstToolCmd implements Runnable {
+
+ @Option(type = OptionType.GLOBAL, title = "storage group number", name = {"-sg",
+ "--storagegroup"}, description = "Storage group number")
+ private String sgNumString = "10";
+
+ @Option(type = OptionType.GLOBAL, title = "total timeseries number", name = {"-ts",
+ "--tsNum"}, description = "Total timeseries number")
+ private String tsNumString = "1000";
+
+ @Option(title = "max timeseries", name = {"-mts",
+ "--mtsNum"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number")
+ private String maxTsNumString = "0";
+
+ @Override
+ public void run() {
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ long memTableSize = config.getMemtableSizeThreshold();
+ int maxMemtableNumber = config.getMaxMemtableNumber();
+ long tsFileSize = config.getTsFileSizeThreshold();
+ long memory = IoTDBConstant.GB;
+ int sgNum = Integer.parseInt(sgNumString);
+ int tsNum = Integer.parseInt(tsNumString);
+ int maxTsNum = Integer.parseInt(maxTsNumString);
+ while (true) {
+ // init parameter
+ config.setAllocateMemoryForWrite(memory);
+ config.setMemtableSizeThreshold(memTableSize);
+ config.setMaxMemtableNumber(maxMemtableNumber);
+ config.setTsFileSizeThreshold(tsFileSize);
+ IoTDBConfigDynamicAdapter.getInstance().reset();
+ IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
+ MManager.getInstance().clear();
+
+ int sgCnt = 1;
+ int tsCnt = 1;
+ try {
+ for (; sgCnt <= sgNum; sgCnt++) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
+ }
+ for (; tsCnt <= tsNum; tsCnt++) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
+ MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(
+ maxTsNum == 0 ? tsCnt / sgNum + 1 : Math.min(tsCnt, maxTsNum));
+ }
+
+ } catch (ConfigAdjusterException e) {
+ if(sgCnt > sgNum) {
+ System.out
+ .print(String.format("Memory estimation progress : %d%%\r", tsCnt * 100 / tsNum));
+ }
+ memory += IoTDBConstant.GB;
+ continue;
+ }
+ break;
+ }
+ System.out.println(String.format("SG: %d, TS: %d, MTS: %d, Memory for writing: %dGB", sgNum,
+ tsNum, maxTsNum == 0 ? tsNum / sgNum + 1 : maxTsNum, memory / IoTDBConstant.GB));
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
index d06df82..d390403 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iotdb.db.tools;
import static org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode.WAL_FILE_NAME;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
index 27fe9e3..1222935 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
@@ -146,44 +146,4 @@ public class IoTDBConfigDynamicAdapterTest {
assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
}
}
-
- @Test
- public void addOrDeleteTimeSeriesSyso2() {
- int sgNum = 1;
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- long memTableSize = config.getMemtableSizeThreshold();
- int maxMemtableNumber = config.getMaxMemtableNumber();
- long tsFileSize = config.getTsFileSizeThreshold();
- long memory = 1024 * 1024 * 1024L;
- while (true) {
- config.setAllocateMemoryForWrite(memory);
- config.setMemtableSizeThreshold(memTableSize);
- config.setMaxMemtableNumber(maxMemtableNumber);
- config.setTsFileSizeThreshold(tsFileSize);
- IoTDBConfigDynamicAdapter.getInstance().reset();
- IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
- MManager.getInstance().clear();
- for (int i = 1; i <= 50 ; i++) {
- try {
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum);
- } catch (ConfigAdjusterException e) {
- e.printStackTrace();
- }
- }
- int i = 1;
- try {
- for (; i <= 10000000; i++) {
- IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
- MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 50 + 1);
- }
- } catch (ConfigAdjusterException e) {
-// System.out.println(i);
- memory += 1024 * 1024 * 1024L;
-// System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
- continue;
- }
- break;
- }
- System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
- }
}
\ No newline at end of file