You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/07/17 05:02:59 UTC
[incubator-iotdb] 01/02: fix sonars and add memory estimation method
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch dev_memory_estimate_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f3f4099a32fffb0ea8e6f860dd2db24744b41dfd
Author: lta <li...@163.com>
AuthorDate: Wed Jul 17 10:08:04 2019 +0800
fix sonars and add memory estimation method
---
iotdb/iotdb/bin/memory-tool.sh | 48 ++++++++++++++++++
iotdb/iotdb/bin/momory-tool.bat | 58 ++++++++++++++++++++++
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 1 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +-
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 19 +++----
.../iotdb/db/cost/statistic/Measurement.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 48 ------------------
.../writelog/recover/TsFileRecoverPerformer.java | 5 +-
.../adapter/IoTDBConfigDynamicAdapterTest.java | 41 +++++++++++++++
pom.xml | 5 ++
11 files changed, 168 insertions(+), 73 deletions(-)
diff --git a/iotdb/iotdb/bin/memory-tool.sh b/iotdb/iotdb/bin/memory-tool.sh
new file mode 100755
index 0000000..c28fff6
--- /dev/null
+++ b/iotdb/iotdb/bin/memory-tool.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+
+MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib_cluster/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+
+exit $?
diff --git a/iotdb/iotdb/bin/momory-tool.bat b/iotdb/iotdb/bin/momory-tool.bat
new file mode 100755
index 0000000..2387a9e
--- /dev/null
+++ b/iotdb/iotdb/bin/momory-tool.bat
@@ -0,0 +1,58 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -DIOTDB_HOME=%IOTDB_HOME%
+
+REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib_cluster\*.jar") do call :append "%%i"
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+REM -----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java b/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
index ef44a92..f79851b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
@@ -26,7 +26,6 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 325dd6c..ecf617b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -211,7 +211,7 @@ public class IoTDBConfig {
/**
* The memory used for stat performance.
*/
- private int performance_stat_memory_in_kb = 20;
+ private int performanceStatMemoryInKB = 20;
/**
* whether use chunkBufferPool.
*/
@@ -565,12 +565,12 @@ public class IoTDBConfig {
this.performanceStatDisplayInterval = performanceStatDisplayInterval;
}
- public int getPerformance_stat_memory_in_kb() {
- return performance_stat_memory_in_kb;
+ public int getPerformanceStatMemoryInKB() {
+ return performanceStatMemoryInKB;
}
- public void setPerformance_stat_memory_in_kb(int performance_stat_memory_in_kb) {
- this.performance_stat_memory_in_kb = performance_stat_memory_in_kb;
+ public void setPerformanceStatMemoryInKB(int performanceStatMemoryInKB) {
+ this.performanceStatMemoryInKB = performanceStatMemoryInKB;
}
public long getMemtableSizeThreshold() {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index da95f84..c159598 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -226,9 +226,9 @@ public class IoTDBDescriptor {
conf.setPerformanceStatDisplayInterval(Long
.parseLong(properties.getProperty("performance_stat_display_interval",
Long.toString(conf.getPerformanceStatDisplayInterval())).trim()));
- conf.setPerformance_stat_memory_in_kb(Integer
+ conf.setPerformanceStatMemoryInKB(Integer
.parseInt(properties.getProperty("performance_stat_memory_in_kb",
- Integer.toString(conf.getPerformance_stat_memory_in_kb())).trim()));
+ Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
} catch (IOException e) {
logger.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 5c626d0..3e809e3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.db.conf.adapter;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.exception.ConfigAdjusterException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
@@ -37,10 +36,11 @@ import org.slf4j.LoggerFactory;
*
* 1. maxMemTableNum. This parameter represents the size of the MemTable available in the MemTable
* pool, which is closely related to the number of storage groups. When adding or deleting a storage
- * group, the parameter also adds or deletes two MemTables. The reason why adding or deleting two
+ * group, the parameter also adds or deletes four MemTables. The reason why adding or deleting four
* MemTables is that when the system is running stably, the speed of the flush operation is faster
* than that of data writing, so one is used for the Flush process and the other is used for data
- * writing. Otherwise, the system should limit the speed of data writing to maintain stability.
+ * writing. Otherwise, the system should limit the speed of data writing to maintain stability. And
+ * two for sequence data, two for unsequence data.
*
* 2. memtableSize. This parameter determines the threshold value for the MemTable in memory to be
* flushed into disk. When the system load increases, the parameter should be set smaller so that
@@ -76,15 +76,9 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
// static parameter section
/**
- * When the size of the adjusted MemTable decreases more than this parameter, trigger the global
- * flush operation and flush all MemTable that meets the flush condition to disk.
- */
- private static final double FLUSH_THRESHOLD = 0.2;
-
- /**
* Maximum amount of memory allocated for write process.
*/
- private static final long ALLOCATE_MEMORY_FOR_WRITE = CONFIG.getAllocateMemoryForWrite();
+ private static long allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
/**
* Metadata size of per timeseries, the default value is 2KB.
@@ -168,7 +162,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
// when unit is byte, it's likely to cause Long type overflow.
// so when b is larger than Integer.MAC_VALUE use the unit KB.
double a = ratio * maxMemTableNum;
- double b = (ALLOCATE_MEMORY_FOR_WRITE - staticMemory) * ratio;
+ double b = (allocateMemoryForWrite - staticMemory) * ratio;
int magnification = b > Integer.MAX_VALUE ? 1024 : 1;
b /= magnification;
double c = (double) CONFIG.getTsFileSizeThreshold() * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE
@@ -187,7 +181,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
* @return Tsfile byte threshold
*/
private long calcTsFileSize(long memTableSize) {
- return (long) ((ALLOCATE_MEMORY_FOR_WRITE - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
+ return (long) ((allocateMemoryForWrite - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
.getInstance().getRatio()
* memTableSize / (maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE * MManager.getInstance()
.getMaximalSeriesNumberAmongStorageGroups()));
@@ -253,6 +247,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
totalTimeseries = 0;
staticMemory = 0;
maxMemTableNum = MEM_TABLE_AVERAGE_QUEUE_LEN;
+ allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
initialized = false;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
index 5af3833..1c7aded 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
@@ -120,7 +120,7 @@ public class Measurement implements MeasurementMBean, IService {
IoTDBConfig tdbConfig = IoTDBDescriptor.getInstance().getConfig();
isEnableStat = tdbConfig.isEnablePerformanceStat();
displayIntervalInMs = tdbConfig.getPerformanceStatDisplayInterval();
- int memoryInKb = tdbConfig.getPerformance_stat_memory_in_kb();
+ int memoryInKb = tdbConfig.getPerformanceStatMemoryInKB();
queueSize = memoryInKb * 1000 / Operation.values().length / 8;
operationLatenciesQueue = new ConcurrentCircularArray[Operation.values().length];
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 5702ecb..d25e913 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -180,54 +180,6 @@ public abstract class AbstractMemTable implements IMemTable {
this.modifications.add(deletion);
}
- /**
- * If chunk contains data with timestamp less than 'timestamp', create a copy and delete all those
- * data. Otherwise return null.
- *
- * @param chunk the source chunk.
- * @param timestamp the upper-bound of deletion time.
- * @return A reduced copy of chunk if chunk contains data with timestamp less than 'timestamp', of
- * null.
- */
- private IWritableMemChunk filterChunk(IWritableMemChunk chunk, long timestamp) {
-
- if (!chunk.isEmpty() && chunk.getMinTime() <= timestamp) {
- //TODO we can avoid sorting data here by scanning data once.
- List<TimeValuePair> timeValuePairs = chunk.getSortedTimeValuePairList();
- TSDataType dataType = chunk.getType();
- IWritableMemChunk newChunk = genMemSeries(dataType);
- for (TimeValuePair pair : timeValuePairs) {
- if (pair.getTimestamp() > timestamp) {
- switch (dataType) {
- case BOOLEAN:
- newChunk.putBoolean(pair.getTimestamp(), pair.getValue().getBoolean());
- break;
- case DOUBLE:
- newChunk.putDouble(pair.getTimestamp(), pair.getValue().getDouble());
- break;
- case INT64:
- newChunk.putLong(pair.getTimestamp(), pair.getValue().getLong());
- break;
- case INT32:
- newChunk.putInt(pair.getTimestamp(), pair.getValue().getInt());
- break;
- case FLOAT:
- newChunk.putFloat(pair.getTimestamp(), pair.getValue().getFloat());
- break;
- case TEXT:
- newChunk.putBinary(pair.getTimestamp(), pair.getValue().getBinary());
- break;
- default:
- throw new UnsupportedOperationException("Unknown datatype: " + dataType);
- }
- }
- }
- TVListAllocator.getInstance().release(dataType, chunk.getTVList());
- return newChunk;
- }
- return null;
- }
-
public void setVersion(long version) {
this.version = version;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 4eff190..c6dc976 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -25,14 +25,11 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
-import java.util.stream.Collectors;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.MemTableFlushTask;
import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
index 2958ab6..27fe9e3 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
@@ -131,6 +131,7 @@ public class IoTDBConfigDynamicAdapterTest {
MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 30 + 1);
}
} catch (ConfigAdjusterException e) {
+ System.out.println(i);
assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
}
int j =0;
@@ -145,4 +146,44 @@ public class IoTDBConfigDynamicAdapterTest {
assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
}
}
+
+ @Test
+ public void addOrDeleteTimeSeriesSyso2() {
+ int sgNum = 1;
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ long memTableSize = config.getMemtableSizeThreshold();
+ int maxMemtableNumber = config.getMaxMemtableNumber();
+ long tsFileSize = config.getTsFileSizeThreshold();
+ long memory = 1024 * 1024 * 1024L;
+ while (true) {
+ config.setAllocateMemoryForWrite(memory);
+ config.setMemtableSizeThreshold(memTableSize);
+ config.setMaxMemtableNumber(maxMemtableNumber);
+ config.setTsFileSizeThreshold(tsFileSize);
+ IoTDBConfigDynamicAdapter.getInstance().reset();
+ IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
+ MManager.getInstance().clear();
+ for (int i = 1; i <= 50 ; i++) {
+ try {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum);
+ } catch (ConfigAdjusterException e) {
+ e.printStackTrace();
+ }
+ }
+ int i = 1;
+ try {
+ for (; i <= 10000000; i++) {
+ IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
+ MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 50 + 1);
+ }
+ } catch (ConfigAdjusterException e) {
+// System.out.println(i);
+ memory += 1024 * 1024 * 1024L;
+// System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
+ continue;
+ }
+ break;
+ }
+ System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
+ }
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 9c77be2..17cad2c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,11 @@
<artifactId>jaxb-runtime</artifactId>
<version>2.4.0-b180725.0644</version>
</dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>airline</artifactId>
+ <version>0.8</version>
+ </dependency>
</dependencies>
<modules>
<module>tsfile</module>