You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/06/25 13:32:15 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated: add
primitive data list pool and reuse primitive data list
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature_async_close_tsfile by this push:
new 8b099ba add primitive data list pool and reuse primitive data list
new 011a9f9 Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
8b099ba is described below
commit 8b099ba4df1eb238744b2be3bd2fe64f660c1564
Author: lta <li...@163.com>
AuthorDate: Tue Jun 25 21:31:23 2019 +0800
add primitive data list pool and reuse primitive data list
---
iotdb/iotdb/conf/iotdb-engine.properties | 24 ++---
iotdb/iotdb/conf/tsfile-format.properties | 10 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 6 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 14 +--
.../db/concurrent/IoTDBThreadPoolFactory.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +--
.../engine/bufferwrite/BufferWriteProcessor.java | 14 ++-
.../bufferwrite/RestorableTsFileIOWriter.java | 4 +-
.../engine/cache/RowGroupBlockMetaDataCache.java | 2 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 44 ++++-----
.../db/engine/filenode/FileNodeProcessor.java | 42 ++++----
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 8 +-
.../iotdb/db/engine/filenodeV2/FlushManager.java | 2 +-
.../filenodeV2/UnsealedTsFileProcessorV2.java | 2 +-
.../db/engine/memcontrol/BasicMemController.java | 2 +-
.../db/engine/memcontrol/FlushPartialPolicy.java | 2 +-
.../db/engine/memcontrol/ForceFLushAllPolicy.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 7 +-
.../db/engine/memtable/IWritableMemChunk.java | 2 +
.../iotdb/db/engine/memtable/MemTablePool.java | 10 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 19 ++--
.../apache/iotdb/db/monitor/MonitorConstants.java | 2 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 4 +-
.../iotdb/db/monitor/collector/FileSize.java | 12 +--
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 6 +-
.../groupby/GroupByWithValueFilterDataSet.java | 6 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../java/org/apache/iotdb/db/service/Monitor.java | 2 +-
.../java/org/apache/iotdb/db/service/Utils.java | 4 +-
.../org/apache/iotdb/db/sync/conf/Constans.java | 2 +-
.../java/org/apache/iotdb/db/utils/AuthUtils.java | 8 +-
.../java/org/apache/iotdb/db/utils/FileUtils.java | 2 +-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 6 +-
.../apache/iotdb/db/utils/PrimitiveArrayList.java | 4 +-
.../iotdb/db/utils/PrimitiveArrayListFactory.java | 1 +
...iveArrayList.java => PrimitiveArrayListV2.java} | 87 ++++++++---------
.../iotdb/db/utils/PrimitiveDataListPool.java | 107 +++++++++++++++++++++
.../org/apache/iotdb/db/utils/TsPrimitiveType.java | 4 +-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 2 +-
.../iotdb/db/writelog/io/SingleFileLogReader.java | 2 +-
.../iotdb/db/monitor/collector/FileSizeTest.java | 4 +-
.../org/apache/iotdb/db/sql/SQLParserTest.java | 2 +-
.../org/apache/iotdb/db/sync/test/RandomNum.java | 2 +-
...ListTest.java => PrimitiveArrayListV2Test.java} | 54 ++++++++++-
...istTest.java => PrimitiveDataListPoolTest.java} | 34 ++++---
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 4 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 2 +-
47 files changed, 382 insertions(+), 215 deletions(-)
diff --git a/iotdb/iotdb/conf/iotdb-engine.properties b/iotdb/iotdb/conf/iotdb-engine.properties
index 66d0cf2..333f621 100644
--- a/iotdb/iotdb/conf/iotdb-engine.properties
+++ b/iotdb/iotdb/conf/iotdb-engine.properties
@@ -53,7 +53,7 @@ force_wal_period_in_ms=10
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/data).
# If it is absolute, system will save the data in exact location it points to.
# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If data_dir is assigned an empty string(i.e.,zero-length), it will be handled as a relative path.
+# Note: If data_dir is assigned an empty string(i.e.,zero-getTotalDataNumber), it will be handled as a relative path.
# For windows platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
# data_dir=D:\\iotdb\\data\\data
@@ -66,7 +66,7 @@ force_wal_period_in_ms=10
# If this property is unset, system will save the TsFiles in the default relative path directory under the data_dir folder(i.e., %IOTDB_HOME%/data/data/settled).
# If some are absolute, system will save the data in exact location they point to.
# If some are relative, system will save the data in the relative path directory they indicate under the data_dir folder.
-# Note: If some are assigned an empty string(i.e.,zero-length), they will be handled as a relative path.
+# Note: If some are assigned an empty string(i.e.,zero-getTotalDataNumber), they will be handled as a relative path.
# For windows platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
# tsfile_dir=D:\\iotdb\\data\\data,data\\data
@@ -79,7 +79,7 @@ force_wal_period_in_ms=10
# If this property is unset, system will save the Overflow Files in the default relative path directory under the data_dir folder(i.e., %IOTDB_HOME%/data/data/overflow).
# If some are absolute, system will save the data in exact location they point to.
# If some are relative, system will save the data in the relative path directory they indicate under the data_dir folder.
-# Note: If some are assigned an empty string(i.e.,zero-length), they will be handled as a relative path.
+# Note: If some are assigned an empty string(i.e.,zero-getTotalDataNumber), they will be handled as a relative path.
# For windows platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
# overflow_dir=D:\\iotdb\\data\\unseq,overflow2
@@ -103,7 +103,7 @@ force_wal_period_in_ms=10
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/system).
# If it is absolute, system will save the data in exact location it points to.
# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If sys_dir is assigned an empty string(i.e.,zero-length), it will be handled as a relative path.
+# Note: If sys_dir is assigned an empty string(i.e.,zero-getTotalDataNumber), it will be handled as a relative path.
# For windows platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
# sys_dir=D:\\iotdb\\data\\system
@@ -115,7 +115,7 @@ force_wal_period_in_ms=10
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data).
# If it is absolute, system will save the data in the exact location it points to.
# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If wal_dir is assigned an empty string(i.e.,zero-length), it will be handled as a relative path.
+# Note: If wal_dir is assigned an empty string(i.e.,zero-getTotalDataNumber), it will be handled as a relative path.
# For windows platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
# wal_dir=D:\\iotdb\\data
@@ -174,21 +174,21 @@ mem_threshold_dangerous=0.6
mem_monitor_interval_in_ms=1000
# Decide how to control memory used by inserting data.
-# 0 is RecordMemController, which count the size of every record (tuple).
+# 0 is RecordMemController, which count the getTotalDataNumber of every record (tuple).
# 1 is JVMMemController, which use JVM heap memory as threshold.
# 2 is DisabledMemController, which does not control memory usage.
mem_controller_type=0
-# When a bufferwrite's metadata size (in byte) exceed this, the bufferwrite is forced closed.
+# When a bufferwrite's metadata getTotalDataNumber (in byte) exceed this, the bufferwrite is forced closed.
bufferwrite_meta_size_threshold=209715200
-# When a bufferwrite's file size (in byte) exceeds this, the bufferwrite is forced closed.
+# When a bufferwrite's file getTotalDataNumber (in byte) exceeds this, the bufferwrite is forced closed.
bufferwrite_file_size_threshold=2147483648
-# When a overflow's metadata size (in byte) exceeds this, the overflow is forced closed.
+# When a overflow's metadata getTotalDataNumber (in byte) exceeds this, the overflow is forced closed.
overflow_meta_size_threshold=20971520
-# When a overflow's file size (in byte) exceeds this, the overflow is forced closed.
+# When a overflow's file getTotalDataNumber (in byte) exceeds this, the overflow is forced closed.
overflow_file_size_threshold=209715200
# How many thread can concurrently flush. When <= 0, use CPU core number.
@@ -229,11 +229,11 @@ small_flush_interval=60000
# The threshold of lines of external sort
external_sort_threshold=50
-# cache size for MManager.
+# cache getTotalDataNumber for MManager.
# This cache is used to improve insert speed where all path check and TSDataType will be cached in MManager with corresponding Path.
schema_manager_cache_size=300000
-# If the size in byte of a WAL log exceeds this, it won't be written.
+# If the getTotalDataNumber in byte of a WAL log exceeds this, it won't be written.
# Generally the default value 4MB is enough.
max_log_entry_size=4194304
diff --git a/iotdb/iotdb/conf/tsfile-format.properties b/iotdb/iotdb/conf/tsfile-format.properties
index 7148006..17593c7 100644
--- a/iotdb/iotdb/conf/tsfile-format.properties
+++ b/iotdb/iotdb/conf/tsfile-format.properties
@@ -18,23 +18,23 @@
#
group_size_in_byte=134217728
-# The memory size for each series writer to pack page, default value is 64KB
+# The memory getTotalDataNumber for each series writer to pack page, default value is 64KB
page_size_in_byte=65536
# The maximum number of data points in a page, defalut 1024*1024
max_number_of_points_in_page=1048576
# Data type configuration
# Data type for input timestamp, TsFile supports INT32 or INT64
time_series_data_type=INT64
-# Max length limitation of input string
+# Max getTotalDataNumber limitation of input string
max_string_length=128
# Floating-point precision
float_precision=2
# Encoder configuration
-# Encoder of time series, TsFile supports TS_2DIFF, PLAIN and RLE(run-length encoding) and default value is TS_2DIFF
+# Encoder of time series, TsFile supports TS_2DIFF, PLAIN and RLE(run-getTotalDataNumber encoding) and default value is TS_2DIFF
time_series_encoder=TS_2DIFF
# Encoder of value series. default value is PLAIN.
-# For int, long data type, TsFile also supports TS_2DIFF and RLE(run-length encoding).
-# For float, double data type, TsFile also supports TS_2DIFF, RLE(run-length encoding) and GORILLA.
+# For int, long data type, TsFile also supports TS_2DIFF and RLE(run-getTotalDataNumber encoding).
+# For float, double data type, TsFile also supports TS_2DIFF, RLE(run-getTotalDataNumber encoding) and GORILLA.
# For text data type, TsFile only supports PLAIN.
value_encoder=PLAIN
# Compression configuration
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java b/iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
index b8f8f64..66d593f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java
@@ -37,11 +37,11 @@ import org.apache.iotdb.db.utils.IOUtils;
/**
* This class store each role in a separate sequential file. Role file schema : Int32 role name
- * length Utf-8 role name bytes Int32 seriesPath privilege number n Int32 seriesPath[1] length Utf-8
+ * getTotalDataNumber Utf-8 role name bytes Int32 seriesPath privilege number n Int32 seriesPath[1] getTotalDataNumber Utf-8
* seriesPath[1] bytes Int32 privilege num k1 Int32 privilege[1][1] Int32 privilege[1][2] ... Int32
- * privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2] bytes Int32 privilege num yk2
+ * privilege[1][k1] Int32 seriesPath[2] getTotalDataNumber Utf-8 seriesPath[2] bytes Int32 privilege num yk2
* Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32 seriesPath[n]
- * length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
+ * getTotalDataNumber Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
* privilege[n][2] ... Int32 privilege[n][kn]
*/
public class LocalFileRoleAccessor implements IRoleAccessor {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java b/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
index b3020ad..e25f99f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
@@ -39,15 +39,15 @@ import org.slf4j.LoggerFactory;
/**
* This class loads a user's information from the corresponding file.The user file is a sequential
- * file. User file schema: Int32 username bytes length Utf-8 username bytes Int32 Password bytes
- * length Utf-8 password bytes Int32 seriesPath privilege number n Int32 seriesPath[1] length Utf-8
+ * file. User file schema: Int32 username bytes getTotalDataNumber Utf-8 username bytes Int32 Password bytes
+ * getTotalDataNumber Utf-8 password bytes Int32 seriesPath privilege number n Int32 seriesPath[1] getTotalDataNumber Utf-8
* seriesPath[1] bytes Int32 privilege num k1 Int32 privilege[1][1] Int32 privilege[1][2] ... Int32
- * privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2] bytes Int32 privilege num k2
+ * privilege[1][k1] Int32 seriesPath[2] getTotalDataNumber Utf-8 seriesPath[2] bytes Int32 privilege num k2
* Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32 seriesPath[n]
- * length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
- * privilege[n][2] ... Int32 privilege[n][kn] Int32 user name number m Int32 user name[1] length
- * Utf-8 user name[1] bytes Int32 user name[2] length Utf-8 user name[2] bytes ... Int32 user
- * name[m] length Utf-8 user name[m] bytes
+ * getTotalDataNumber Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
+ * privilege[n][2] ... Int32 privilege[n][kn] Int32 user name number m Int32 user name[1] getTotalDataNumber
+ * Utf-8 user name[1] bytes Int32 user name[2] getTotalDataNumber Utf-8 user name[2] bytes ... Int32 user
+ * name[m] getTotalDataNumber Utf-8 user name[m] bytes
*/
public class LocalFileUserAccessor implements IUserAccessor {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/concurrent/IoTDBThreadPoolFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/concurrent/IoTDBThreadPoolFactory.java
index 7399903..545ade0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/concurrent/IoTDBThreadPoolFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/concurrent/IoTDBThreadPoolFactory.java
@@ -37,7 +37,7 @@ public class IoTDBThreadPoolFactory {
* see {@link Executors#newFixedThreadPool(int, java.util.concurrent.ThreadFactory)}.
*
* @param poolName - the name of thread pool
- * @return fixed size thread pool
+ * @return fixed getTotalDataNumber thread pool
*/
public static ExecutorService newFixedThreadPool(int nthreads, String poolName) {
return Executors.newFixedThreadPool(nthreads, new IoTThreadFactory(poolName));
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c3d2c88..bb27bf3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -191,24 +191,24 @@ public class IoTDBConfig {
private long memMonitorInterval = 1000;
/**
* Decide how to control memory usage of inserting data. 0 is RecordMemController, which sums the
- * size of each record (tuple). 1 is JVMMemController, which uses the JVM heap memory as the
+ * getTotalDataNumber of each record (tuple). 1 is JVMMemController, which uses the JVM heap memory as the
* memory usage indicator.
*/
private int memControllerType = 1;
/**
- * When a bufferwrite's metadata size (in byte) exceed this, the bufferwrite is forced closed.
+ * When a bufferwrite's metadata getTotalDataNumber (in byte) exceed this, the bufferwrite is forced closed.
*/
private long bufferwriteMetaSizeThreshold = 200 * 1024 * 1024L;
/**
- * When a bufferwrite's file size (in byte) exceed this, the bufferwrite is forced closed.
+ * When a bufferwrite's file getTotalDataNumber (in byte) exceed this, the bufferwrite is forced closed.
*/
private long bufferwriteFileSizeThreshold = 2 * 1024 * 1024 * 1024L;
/**
- * When a overflow's metadata size (in byte) exceed this, the overflow is forced closed.
+ * When a overflow's metadata getTotalDataNumber (in byte) exceed this, the overflow is forced closed.
*/
private long overflowMetaSizeThreshold = 20 * 1024 * 1024L;
/**
- * When a overflow's file size (in byte) exceed this, the overflow is forced closed.
+ * When a overflow's file getTotalDataNumber (in byte) exceed this, the overflow is forced closed.
*/
private long overflowFileSizeThreshold = 200 * 1024 * 1024L;
/**
@@ -248,11 +248,11 @@ public class IoTDBConfig {
*/
private int externalSortThreshold = 50;
/**
- * Cache size of {@code checkAndGetDataTypeCache} in {@link MManager}.
+ * Cache getTotalDataNumber of {@code checkAndGetDataTypeCache} in {@link MManager}.
*/
private int mManagerCacheSize = 400000;
/**
- * The maximum size of a single log in byte. If a log exceeds this size, it cannot be written to
+ * The maximum getTotalDataNumber of a single log in byte. If a log exceeds this getTotalDataNumber, it cannot be written to
* the WAL file and an exception is thrown.
*/
private int maxLogEntrySize = 4 * 1024 * 1024;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
index 243c43e..35765e7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
@@ -18,8 +18,6 @@
*/
package org.apache.iotdb.db.engine.bufferwrite;
-import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR;
-
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
@@ -177,7 +175,7 @@ public class BufferWriteProcessor extends Processor {
* @param timestamp timestamp of the data point
* @param dataType the data type of the value
* @param value data point value
- * @return true -the size of tsfile or metadata reaches to the threshold. false -otherwise
+ * @return true -the getTotalDataNumber of tsfile or metadata reaches to the threshold. false -otherwise
* @throws BufferWriteProcessorException if a flushing operation occurs and failed.
*/
public boolean write(String deviceId, String measurementId, long timestamp, TSDataType dataType,
@@ -444,7 +442,7 @@ public class BufferWriteProcessor extends Processor {
} else {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(
- "Begin to submit flush task for bufferwrite processor {}, current Flush Queue is {}, core pool size is {}.",
+ "Begin to submit flush task for bufferwrite processor {}, current Flush Queue is {}, core pool getTotalDataNumber is {}.",
getProcessorName(), FlushPoolManager.getInstance().getWaitingTasksNumber(),
FlushPoolManager.getInstance().getCorePoolSize());
}
@@ -532,9 +530,9 @@ public class BufferWriteProcessor extends Processor {
}
/**
- * get metadata size.
+ * get metadata getTotalDataNumber.
*
- * @return The sum of all timeseries's metadata size within this file.
+ * @return The sum of all timeseries's metadata getTotalDataNumber within this file.
*/
public long getMetaSize() {
// TODO : [MemControl] implement this
@@ -542,9 +540,9 @@ public class BufferWriteProcessor extends Processor {
}
/**
- * get file size.
+ * get file getTotalDataNumber.
*
- * @return The file size of the TsFile corresponding to this processor.
+ * @return The file getTotalDataNumber of the TsFile corresponding to this processor.
*/
public long getFileSize() {
// TODO : save this variable to avoid object creation?
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
index d231701..ec5b109 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
@@ -108,7 +108,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
recoverMetadata(existedMetadatas);
LOGGER.info(
"Recover the bufferwrite processor {}, the tsfile seriesPath is {}, "
- + "the position of last flushed ChunkGroup is {}, the size of rowGroupMetadata is {}",
+ + "the position of last flushed ChunkGroup is {}, the getTotalDataNumber of rowGroupMetadata is {}",
processorName, insertFilePath, position, existedMetadatas.size());
isNewResource = false;
} else {
@@ -165,7 +165,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
tsDeviceMetadata.serializeTo(baos);
- // insert metadata size using int
+ // insert metadata getTotalDataNumber using int
int metadataSize = baos.size();
out.write(BytesUtils.intToBytes(metadataSize));
// insert metadata
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
index 044fb5d..6be0958 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
@@ -104,7 +104,7 @@ public class RowGroupBlockMetaDataCache {
}
/**
- * the default LRU cache size is 100. The singleton pattern.
+ * the default LRU cache getTotalDataNumber is 100. The singleton pattern.
*/
private static class RowGroupBlockMetaDataCacheSingleton {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index c93d295..8247157 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@ -110,7 +110,7 @@
// }
//
// String normalizedBaseDir = baseDir;
-// if (normalizedBaseDir.charAt(normalizedBaseDir.length() - 1) != File.separatorChar) {
+// if (normalizedBaseDir.charAt(normalizedBaseDir.getTotalDataNumber() - 1) != File.separatorChar) {
// normalizedBaseDir += Character.toString(File.separatorChar);
// }
// this.baseDir = normalizedBaseDir;
@@ -126,12 +126,12 @@
// }
//
// closedProcessorCleaner.scheduleWithFixedDelay(()->{
-// int size = 0;
+// int getTotalDataNumber = 0;
// for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
-// size += fileNodeProcessor.getClosingBufferWriteProcessor().size();
+// getTotalDataNumber += fileNodeProcessor.getClosingBufferWriteProcessor().getTotalDataNumber();
// }
-// if (size > 5) {
-// LOGGER.info("Current closing processor number is {}", size);
+// if (getTotalDataNumber > 5) {
+// LOGGER.info("Current closing processor number is {}", getTotalDataNumber);
// }
//// for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
//// fileNodeProcessor.checkAllClosingProcessors();
@@ -148,7 +148,7 @@
// statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_FAIL.name())
// .incrementAndGet();
// statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_FAIL.name())
-// .addAndGet(tsRecord.dataPointList.size());
+// .addAndGet(tsRecord.dataPointList.getTotalDataNumber());
// }
//
// /**
@@ -337,7 +337,7 @@
// if (!isMonitor) {
// fileNodeProcessor.getStatParamsHashMap()
// .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
-// .addAndGet(tsRecord.dataPointList.size());
+// .addAndGet(tsRecord.dataPointList.getTotalDataNumber());
// fileNodeProcessor.getStatParamsHashMap()
// .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
// .incrementAndGet();
@@ -345,7 +345,7 @@
// .incrementAndGet();
// statParamsHashMap
// .get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
-// .addAndGet(tsRecord.dataPointList.size());
+// .addAndGet(tsRecord.dataPointList.getTotalDataNumber());
// }
// return insertType;
// }
@@ -354,8 +354,8 @@
// throws FileNodeManagerException {
// try {
// if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
-// String[] measurementList = new String[tsRecord.dataPointList.size()];
-// String[] insertValues = new String[tsRecord.dataPointList.size()];
+// String[] measurementList = new String[tsRecord.dataPointList.getTotalDataNumber()];
+// String[] insertValues = new String[tsRecord.dataPointList.getTotalDataNumber()];
// int i=0;
// for (DataPoint dp : tsRecord.dataPointList) {
// measurementList[i] = dp.getMeasurementId();
@@ -383,7 +383,7 @@
// private void updateStat(boolean isMonitor, TSRecord tsRecord) {
// if (!isMonitor) {
// statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS.name())
-// .addAndGet(tsRecord.dataPointList.size());
+// .addAndGet(tsRecord.dataPointList.getTotalDataNumber());
// }
// }
//
@@ -539,7 +539,7 @@
// if (LOGGER.isInfoEnabled()) {
// LOGGER.info(
// "The filenode processor {} will setCloseMark the bufferwrite processor, "
-// + "because the size[{}] of tsfile {} reaches the threshold {}",
+// + "because the getTotalDataNumber[{}] of tsfile {} reaches the threshold {}",
// filenodeName, MemUtils.bytesCntToStr(bufferWriteProcessor.getFileSize()),
// bufferWriteProcessor.getInsertFilePath(), MemUtils.bytesCntToStr(
// IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold()));
@@ -1018,9 +1018,9 @@
//
// private String standardizeDir(String originalPath) {
// String res = originalPath;
-// if ((originalPath.length() > 0
-// && originalPath.charAt(originalPath.length() - 1) != File.separatorChar)
-// || originalPath.length() == 0) {
+// if ((originalPath.getTotalDataNumber() > 0
+// && originalPath.charAt(originalPath.getTotalDataNumber() - 1) != File.separatorChar)
+// || originalPath.getTotalDataNumber() == 0) {
// res = originalPath + File.separatorChar;
// }
// return res;
@@ -1182,7 +1182,7 @@
// // you may add some delicate process like below
// // or you could provide multiple methods for different urgency
// switch (level) {
-// // only select the most urgent (most active or biggest in size)
+// // only select the most urgent (most active or biggest in getTotalDataNumber)
// // processors to flush
// // only select top 10% active memory user to flush
// case WARNING:
@@ -1237,10 +1237,10 @@
// // sort the tempProcessors as descending order
// tempProcessors.sort((o1, o2) -> (int) (o2.memoryUsage() - o1.memoryUsage()));
// int flushNum =
-// (int) (tempProcessors.size() * percentage) > 1
-// ? (int) (tempProcessors.size() * percentage)
+// (int) (tempProcessors.getTotalDataNumber() * percentage) > 1
+// ? (int) (tempProcessors.getTotalDataNumber() * percentage)
// : 1;
-// for (int i = 0; i < flushNum && i < tempProcessors.size(); i++) {
+// for (int i = 0; i < flushNum && i < tempProcessors.getTotalDataNumber(); i++) {
// FileNodeProcessor processor = tempProcessors.get(i);
// // 64M
// if (processor.memoryUsage() <= TSFileConfig.groupSizeInByte / 2) {
@@ -1276,11 +1276,11 @@
//
// boolean notFinished = true;
// while (notFinished) {
-// int size = 0;
+// int getTotalDataNumber = 0;
// for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
-// size += fileNodeProcessor.getClosingBufferWriteProcessor().size();
+// getTotalDataNumber += fileNodeProcessor.getClosingBufferWriteProcessor().getTotalDataNumber();
// }
-// if (size == 0) {
+// if (getTotalDataNumber == 0) {
// notFinished = false;
// } else {
// try {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
index 351bb15..f639014 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
@@ -286,8 +286,8 @@
//
// this.parameters = new HashMap<>();
// String dirPath = fileNodeDirPath;
-// if (dirPath.length() > 0
-// && dirPath.charAt(dirPath.length() - 1) != File.separatorChar) {
+// if (dirPath.getTotalDataNumber() > 0
+// && dirPath.charAt(dirPath.getTotalDataNumber() - 1) != File.separatorChar) {
// dirPath = dirPath + File.separatorChar;
// }
//
@@ -467,7 +467,7 @@
// parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
// parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fileNodeFlushAction);
//
-// for (int i = 0; i < newFileNodes.size(); i++) {
+// for (int i = 0; i < newFileNodes.getTotalDataNumber(); i++) {
// TsFileResource tsFile = newFileNodes.get(i);
//// try {
//// String filePath = tsFile.getFilePath();
@@ -722,7 +722,7 @@
// private int searchIndexNodeByTimestamp(String deviceId, long timestamp,
// List<TsFileResource> fileList) {
// int index = 1;
-// while (index < fileList.size()) {
+// while (index < fileList.getTotalDataNumber()) {
// if (timestamp < fileList.get(index).getStartTime(deviceId)) {
// break;
// } else {
@@ -813,15 +813,15 @@
// // bufferwrite data
// UnsealedTsFile unsealedTsFile = null;
//
-// if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()
-// && !newFileNodes.get(newFileNodes.size() - 1).getStartTimeMap().isEmpty()) {
+// if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).isClosed()
+// && !newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).getStartTimeMap().isEmpty()) {
// unsealedTsFile = new UnsealedTsFile();
-// unsealedTsFile.setFilePath(newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath());
+// unsealedTsFile.setFilePath(newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).getFile().getAbsolutePath());
// if (bufferWriteProcessor == null) {
// throw new FileNodeProcessorException(String.format(
// "The last of tsfile %s in filenode processor %s is not closed, "
// + "but the bufferwrite processor is null.",
-// newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath(), getProcessorName()));
+// newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).getFile().getAbsolutePath(), getProcessorName()));
// }
// bufferwritedata = bufferWriteProcessor
// .queryBufferWriteData(deviceId, measurementId, dataType, mSchema.getProps());
@@ -975,7 +975,7 @@
// .getConfig().getOverflowFileSizeThreshold()) {
// if (LOGGER.isInfoEnabled()) {
// LOGGER.info(
-// "Skip this merge taks submission, because the size{} of overflow processor {} "
+// "Skip this merge taks submission, because the getTotalDataNumber{} of overflow processor {} "
// + "does not reaches the threshold {}.",
// MemUtils.bytesCntToStr(overflowProcessor.getFileSize()), getProcessorName(),
// MemUtils.bytesCntToStr(
@@ -1118,7 +1118,7 @@
//
// // query tsfile data and overflow data, and merge them
// int numOfMergeFiles = 0;
-// int allNeedMergeFiles = backupIntervalFiles.size();
+// int allNeedMergeFiles = backupIntervalFiles.getTotalDataNumber();
// for (TsFileResource backupIntervalFile : backupIntervalFiles) {
// numOfMergeFiles++;
// if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.CHANGED) {
@@ -1234,7 +1234,7 @@
// for (String deviceId : tsFileResource.getEndTimeMap().keySet()) {
// List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
// int index = temp.indexOf(tsFileResource);
-// int size = temp.size();
+// int getTotalDataNumber = temp.getTotalDataNumber();
// // start time
// if (index == 0) {
// startTimeMap.put(deviceId, 0L);
@@ -1242,7 +1242,7 @@
// startTimeMap.put(deviceId, tsFileResource.getStartTime(deviceId));
// }
// // end time
-// if (index < size - 1) {
+// if (index < getTotalDataNumber - 1) {
// endTimeMap.put(deviceId, temp.get(index + 1).getStartTime(deviceId) - 1);
// } else {
// endTimeMap.put(deviceId, tsFileResource.getEndTime(deviceId));
@@ -1276,7 +1276,7 @@
// // reconstruct the file index
// addAllFileIntoIndex(backupIntervalFiles);
// // check the merge changed file
-// for (int i = beginIndex; i < backupIntervalFiles.size(); i++) {
+// for (int i = beginIndex; i < backupIntervalFiles.getTotalDataNumber(); i++) {
// TsFileResource newFile = newFileNodes.get(i - beginIndex);
// TsFileResource temp = backupIntervalFiles.get(i);
// if (newFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
@@ -1287,7 +1287,7 @@
// }
// }
// // add new file when merge
-// for (int i = backupIntervalFiles.size() - beginIndex; i < newFileNodes.size(); i++) {
+// for (int i = backupIntervalFiles.getTotalDataNumber() - beginIndex; i < newFileNodes.getTotalDataNumber(); i++) {
// TsFileResource fileNode = newFileNodes.get(i);
// if (fileNode.isClosed()) {
// result.add(fileNode.backUp());
@@ -1386,9 +1386,9 @@
// collectBufferWriteFiles(bufferFiles);
//
// // add the restore file, if the last file is not closed
-// if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
+// if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).isClosed()) {
// String bufferFileRestorePath =
-// newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath() + RESTORE_FILE_SUFFIX;
+// newFileNodes.get(newFileNodes.getTotalDataNumber() - 1).getFile().getAbsolutePath() + RESTORE_FILE_SUFFIX;
// bufferFiles.add(bufferFileRestorePath);
// }
//
@@ -1428,8 +1428,8 @@
// private void collectBufferWriteDirs(List<String> bufferwriteDirPathList,
// List<File> bufferwriteDirList) {
// for (String bufferwriteDirPath : bufferwriteDirPathList) {
-// if (bufferwriteDirPath.length() > 0
-// && bufferwriteDirPath.charAt(bufferwriteDirPath.length() - 1)
+// if (bufferwriteDirPath.getTotalDataNumber() > 0
+// && bufferwriteDirPath.charAt(bufferwriteDirPath.getTotalDataNumber() - 1)
// != File.separatorChar) {
// bufferwriteDirPath = bufferwriteDirPath + File.separatorChar;
// }
@@ -1677,7 +1677,7 @@
// private String constructOutputFilePath(String baseDir, String processorName, String fileName) {
//
// String localBaseDir = baseDir;
-// if (localBaseDir.charAt(localBaseDir.length() - 1) != File.separatorChar) {
+// if (localBaseDir.charAt(localBaseDir.getTotalDataNumber() - 1) != File.separatorChar) {
// localBaseDir = localBaseDir + File.separatorChar + processorName;
// }
// File dataDir = new File(localBaseDir);
@@ -1877,7 +1877,7 @@
//
// synchronized (fileNodeRestoreLock) {
// File restoreFile = new File(fileNodeRestoreFilePath);
-// if (!restoreFile.exists() || restoreFile.length() == 0) {
+// if (!restoreFile.exists() || restoreFile.getTotalDataNumber() == 0) {
// try {
// return new FileNodeProcessorStore(false, new HashMap<>(),
// new TsFileResource(null, false),
@@ -2098,7 +2098,7 @@
// */
// public void waitforAllClosed() throws FileNodeProcessorException {
// close();
-// while (getClosingBufferWriteProcessor().size() != 0) {
+// while (getClosingBufferWriteProcessor().getTotalDataNumber() != 0) {
// checkAllClosingProcessors();
// try {
// Thread.sleep(10);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index 2b3b958..21fae03 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -284,7 +284,7 @@ public class FileNodeProcessorV2 {
latestTimeForEachDevice.put(insertPlan.getDeviceId(), insertPlan.getTime());
}
- // check memtable size and may asyncFlush the workMemtable
+ // check memtable getTotalDataNumber and may asyncFlush the workMemtable
if (unsealedTsFileProcessor.shouldFlush()) {
if (unsealedTsFileProcessor.shouldClose()) {
asyncCloseTsFileProcessor(unsealedTsFileProcessor, sequence);
@@ -326,11 +326,11 @@ public class FileNodeProcessorV2 {
private void asyncCloseTsFileProcessor(UnsealedTsFileProcessorV2 unsealedTsFileProcessor,
boolean sequence) {
- LOGGER.info("The memtable size {} reaches the threshold, async flush it to tsfile: {}",
+ LOGGER.info("The memtable getTotalDataNumber {} reaches the threshold, async flush it to tsfile: {}",
unsealedTsFileProcessor.getWorkMemTableMemory(),
unsealedTsFileProcessor.getTsFileResource().getFile().getAbsolutePath());
- // check file size and may setCloseMark the BufferWrite
+ // check file getTotalDataNumber and may setCloseMark the BufferWrite
if (sequence) {
closingSequenceTsFileProcessor.add(unsealedTsFileProcessor);
workSequenceTsFileProcessor = null;
@@ -342,7 +342,7 @@ public class FileNodeProcessorV2 {
// async close tsfile
unsealedTsFileProcessor.asyncClose();
- LOGGER.info("The file size {} reaches the threshold, async close tsfile: {}.",
+ LOGGER.info("The file getTotalDataNumber {} reaches the threshold, async close tsfile: {}.",
unsealedTsFileProcessor.getTsFileResource().getFileSize(),
unsealedTsFileProcessor.getTsFileResource().getFile().getAbsolutePath());
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
index b971b75..b683085 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
@@ -58,7 +58,7 @@ public class FlushManager {
public Future registerUnsealedTsFileProcessor(UnsealedTsFileProcessorV2 unsealedTsFileProcessor) {
synchronized (unsealedTsFileProcessor) {
if (!unsealedTsFileProcessor.isManagedByFlushManager() && unsealedTsFileProcessor.getFlushingMemTableSize() > 0) {
- LOGGER.info("begin to submit a flush thread, flushing memtable size: {}", unsealedTsFileProcessor.getFlushingMemTableSize());
+ LOGGER.info("begin to submit a flush thread, flushing memtable getTotalDataNumber: {}", unsealedTsFileProcessor.getFlushingMemTableSize());
unsealedTsFileProcessorQueue.add(unsealedTsFileProcessor);
unsealedTsFileProcessor.setManagedByFlushManager(true);
return flushPool.submit(new FlushThread());
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
index d6c860a..087d8a2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
@@ -281,7 +281,7 @@ public class UnsealedTsFileProcessorV2 {
writer.makeMetadataVisible();
flushingMemTables.remove(memTable);
LOGGER.info(
- "flush finished, remove a memtable from flushing list, flushing memtable list size: {}",
+ "flush finished, remove a memtable from flushing list, flushing memtable list getTotalDataNumber: {}",
flushingMemTables.size());
} finally {
flushQueryLock.writeLock().unlock();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/BasicMemController.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/BasicMemController.java
index 220d7bb..71e8ce2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/BasicMemController.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/BasicMemController.java
@@ -153,7 +153,7 @@ public abstract class BasicMemController implements IService {
}
/**
- * Any object (like OverflowProcessor or BufferWriteProcessor) that wants to hold some fixed size
+ * Any object (like OverflowProcessor or BufferWriteProcessor) that wants to hold some fixed getTotalDataNumber
* of memory should call this method to check the returned memory usage level to decide any
* further actions.
* @param user an object that wants some memory as a buffer or anything.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
index 08567fe..bfab3b4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
@@ -38,7 +38,7 @@ public class FlushPartialPolicy implements Policy {
@Override
public void execute() {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Memory reaches {}, current memory size is {}, JVM memory is {}, flushing.",
+ LOGGER.debug("Memory reaches {}, current memory getTotalDataNumber is {}, JVM memory is {}, flushing.",
BasicMemController.getInstance().getCurrLevel(),
MemUtils.bytesCntToStr(BasicMemController.getInstance().getTotalUsage()),
MemUtils.bytesCntToStr(Runtime.getRuntime().totalMemory()
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
index e68e712..930e260 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
@@ -33,7 +33,7 @@ public class ForceFLushAllPolicy implements Policy {
@Override
public void execute() {
if (logger.isInfoEnabled()) {
- logger.info("Memory reaches {}, current memory size is {}, JVM memory is {}, flushing.",
+ logger.info("Memory reaches {}, current memory getTotalDataNumber is {}, JVM memory is {}, flushing.",
BasicMemController.getInstance().getCurrLevel(),
MemUtils.bytesCntToStr(BasicMemController.getInstance().getTotalUsage()),
MemUtils.bytesCntToStr(Runtime.getRuntime().totalMemory()
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index b90570d..cfc2a58 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -108,7 +108,7 @@ public abstract class AbstractMemTable implements IMemTable {
Object value) {
IWritableMemChunk memSeries = createIfNotExistAndGet(deviceId, measurement, dataType);
memSeries.write(insertTime, value);
- // update memory size of current memtable
+ // update memory getTotalDataNumber of current memtable
}
@Override
@@ -129,6 +129,11 @@ public abstract class AbstractMemTable implements IMemTable {
@Override
public void clear() {
+ for (Map<String, IWritableMemChunk> writableMemChunkMap : memTableMap.values()) {
+ for (IWritableMemChunk memChunk : writableMemChunkMap.values()) {
+ memChunk.releasePrimitiveArrayList();
+ }
+ }
memTableMap.clear();
modifications.clear();
memSize = 0;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index a80a079..22a8aff 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -46,4 +46,6 @@ public interface IWritableMemChunk extends TimeValuePairSorter {
TSDataType getType();
void setTimeOffset(long offset);
+
+ void releasePrimitiveArrayList();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTablePool.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTablePool.java
index 4902ca6..a9493d1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTablePool.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTablePool.java
@@ -48,12 +48,12 @@ public class MemTablePool {
synchronized (emptyMemTables) {
if (emptyMemTables.isEmpty() && size < capacity) {
size++;
- LOGGER.info("generated a new memtable for {}, system memtable size: {}, stack size: {}",
+ LOGGER.info("generated a new memtable for {}, system memtable getTotalDataNumber: {}, stack getTotalDataNumber: {}",
applier, size, emptyMemTables.size());
return new PrimitiveMemTable();
} else if (!emptyMemTables.isEmpty()) {
LOGGER
- .info("system memtable size: {}, stack size: {}, then get a memtable from stack for {}",
+ .info("system memtable getTotalDataNumber: {}, stack getTotalDataNumber: {}, then get a memtable from stack for {}",
size, emptyMemTables.size(), applier);
return emptyMemTables.pop();
}
@@ -63,7 +63,7 @@ public class MemTablePool {
while (true) {
if (!emptyMemTables.isEmpty()) {
LOGGER.info(
- "system memtable size: {}, stack size: {}, then get a memtable from stack for {}",
+ "system memtable getTotalDataNumber: {}, stack getTotalDataNumber: {}, then get a memtable from stack for {}",
size, emptyMemTables.size(), applier);
return emptyMemTables.pop();
}
@@ -82,7 +82,7 @@ public class MemTablePool {
memTable.clear();
emptyMemTables.push(memTable);
emptyMemTables.notify();
- LOGGER.info("a memtable returned, stack size {}", emptyMemTables.size());
+ LOGGER.info("a memtable returned, stack getTotalDataNumber {}", emptyMemTables.size());
}
}
@@ -91,7 +91,7 @@ public class MemTablePool {
memTable.clear();
emptyMemTables.push(memTable);
emptyMemTables.notify();
- LOGGER.info("{} return a memtable, stack size {}", storageGroup, emptyMemTables.size());
+ LOGGER.info("{} return a memtable, stack getTotalDataNumber {}", storageGroup, emptyMemTables.size());
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index a17f265..d0f927d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.iotdb.db.utils.PrimitiveArrayList;
-import org.apache.iotdb.db.utils.PrimitiveArrayListFactory;
+import org.apache.iotdb.db.utils.PrimitiveArrayListV2;
+import org.apache.iotdb.db.utils.PrimitiveDataListPool;
import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.db.utils.TsPrimitiveType;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
@@ -33,12 +33,12 @@ import org.apache.iotdb.tsfile.utils.Binary;
public class WritableMemChunk implements IWritableMemChunk {
private TSDataType dataType;
- private PrimitiveArrayList list;
+ private PrimitiveArrayListV2 list;
private long timeOffset = 0;
public WritableMemChunk(TSDataType dataType) {
this.dataType = dataType;
- this.list = PrimitiveArrayListFactory.getByDataType(dataType);
+ this.list = PrimitiveDataListPool.getInstance().getPrimitiveDataListByDataType(dataType);
}
@Override
@@ -126,7 +126,7 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
// TODO: Consider using arrays to sort and remove duplicates
public List<TimeValuePair> getSortedTimeValuePairList() {
- int length = list.size();
+ int length = list.getTotalDataNumber();
Map<Long, TsPrimitiveType> map = new HashMap<>(length, 1.0f);
for (int i = 0; i < length; i++) {
if (list.getTimestamp(i) >= timeOffset) {
@@ -141,12 +141,12 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
public void reset() {
- this.list = PrimitiveArrayListFactory.getByDataType(dataType);
+ this.list = PrimitiveDataListPool.getInstance().getPrimitiveDataListByDataType(dataType);
}
@Override
public long count() {
- return list.size();
+ return list.getTotalDataNumber();
}
@Override
@@ -159,4 +159,9 @@ public class WritableMemChunk implements IWritableMemChunk {
timeOffset = offset;
}
+ @Override
+ public void releasePrimitiveArrayList(){
+ PrimitiveDataListPool.getInstance().release(list);
+ }
+
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
index 129aa91..a337db4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
@@ -36,7 +36,7 @@ public class MonitorConstants {
private static final String FILENODE_MANAGER_CONST = "FILENODE_MANAGER_CONST";
static final String FILE_SIZE_CONST = "FILE_SIZE_CONST";
public static final String MONITOR_PATH_SEPARATOR = ".";
- // statistic for file size statistic module
+ // statistic for file getTotalDataNumber statistic module
private static final String FILE_SIZE = "file_size";
public static final String FILE_SIZE_STORAGE_GROUP_NAME = STAT_STORAGE_GROUP_PREFIX
+ MONITOR_PATH_SEPARATOR + FILE_SIZE;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index bb5f04d..5f3f630 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -154,7 +154,7 @@ public class StatMonitor implements IService {
/**
* register monitor statistics time series metadata into MManager.
*
- * @param hashMap series path and data type pair, for example: [root.stat.file.size.DATA, INT64]
+ * @param hashMap series path and data type pair, for example: [root.stat.file.getTotalDataNumber.DATA, INT64]
*/
public synchronized void registerStatStorageGroup(Map<String, String> hashMap) {
MManager mManager = MManager.getInstance();
@@ -197,7 +197,7 @@ public class StatMonitor implements IService {
/**
* register class which implemented IStatistic interface into statisticMap
*
- * @param path the stat series prefix path, like root.stat.file.size
+ * @param path the stat series prefix path, like root.stat.file.getTotalDataNumber
* @param iStatistic instance of class which implemented IStatistic interface
*/
public void registerStatistics(String path, IStatistic iStatistic) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
index e9ecba2..eb5f5e7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
@@ -47,7 +47,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class is to collect some file size statistics.
+ * This class is to collect some file getTotalDataNumber statistics.
*/
public class FileSize implements IStatistic {
@@ -128,16 +128,16 @@ public class FileSize implements IStatistic {
}
/**
- * Return a map[FileSizeConstants, Long]. The key is the dir type and the value is the dir size in
+ * Return a map[FileSizeConstants, Long]. The key is the dir type and the value is the dir getTotalDataNumber in
* byte.
*
- * @return a map[FileSizeConstants, Long] with the dir type and the dir size in byte
+ * @return a map[FileSizeConstants, Long] with the dir type and the dir getTotalDataNumber in byte
*/
public Map<FileSizeConstants, Long> getFileSizesInByte() {
EnumMap<FileSizeConstants, Long> fileSizes = new EnumMap<>(FileSizeConstants.class);
for (FileSizeConstants kinds : MonitorConstants.FileSizeConstants.values()) {
if (kinds.equals(MonitorConstants.FileSizeConstants.SETTLED)) {
- //sum bufferWriteDirs size
+ //sum bufferWriteDirs getTotalDataNumber
long settledSize = INIT_VALUE_IF_FILE_NOT_EXIST;
for (String bufferWriteDir : config.getBufferWriteDirs()) {
File settledFile = new File(bufferWriteDir);
@@ -145,7 +145,7 @@ public class FileSize implements IStatistic {
try {
settledSize += FileUtils.sizeOfDirectory(settledFile);
} catch (Exception e) {
- LOGGER.error("Meet error while trying to get {} size with dir {} .", kinds,
+ LOGGER.error("Meet error while trying to get {} getTotalDataNumber with dir {} .", kinds,
bufferWriteDir, e);
fileSizes.put(kinds, ABNORMAL_VALUE);
}
@@ -159,7 +159,7 @@ public class FileSize implements IStatistic {
fileSizes.put(kinds, FileUtils.sizeOfDirectory(file));
} catch (Exception e) {
LOGGER
- .error("Meet error while trying to get {} size with dir {} .", kinds,
+ .error("Meet error while trying to get {} getTotalDataNumber with dir {} .", kinds,
kinds.getPath(),
e);
fileSizes.put(kinds, ABNORMAL_VALUE);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index ffa0cdf..a490531 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -116,7 +116,7 @@ public class PhysicalGenerator {
// UpdatePlan updatePlan = new UpdatePlan();
// updatePlan.setValue(update.getValue());
// paths = update.getSelectedPaths();
- // if (paths.size() > 1) {
+ // if (paths.getTotalDataNumber() > 1) {
// throw new LogicalOperatorException("update command, must have and only have one seriesPath:" + paths);
// }
// updatePlan.setPath(paths.get(0));
@@ -317,9 +317,9 @@ public class PhysicalGenerator {
// break;
// }
// }
- // if (valueList.size() == 1) {
+ // if (valueList.getTotalDataNumber() == 1) {
// valueFilter = valueList.get(0);
- // } else if (valueList.size() > 1) {
+ // } else if (valueList.getTotalDataNumber() > 1) {
// valueFilter = new FilterOperator(KW_AND, false);
// valueFilter.setChildren(valueList);
// }
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
index d984d57..356a726 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
@@ -52,7 +52,7 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
private boolean hasCachedTimestamp;
/**
- * group by batch calculation size.
+ * group by batch calculation getTotalDataNumber.
*/
private int timeStampFetchSize;
@@ -135,8 +135,8 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
* construct an array of timestamps for one batch of a group by partition calculating.
*
* @param timestampArray timestamp array
- * @param timeArrayLength the current length of timestamp array
- * @return time array length
+ * @param timeArrayLength the current getTotalDataNumber of timestamp array
+ * @return time array getTotalDataNumber
*/
private int constructTimeArrayForOneCal(long[] timestampArray, int timeArrayLength)
throws IOException {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
index 6ff9741..a0b4859 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
@@ -60,7 +60,7 @@ public class AggregateEngineExecutor {
private IExpression expression;
/**
- * aggregation batch calculation size.
+ * aggregation batch calculation getTotalDataNumber.
**/
private int aggregateFetchSize;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/Monitor.java b/iotdb/src/main/java/org/apache/iotdb/db/service/Monitor.java
index 53fb90f..bd46419 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/Monitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/Monitor.java
@@ -43,7 +43,7 @@ public class Monitor implements MonitorMBean, IService {
try {
return FileUtils.sizeOfDirectory(new File(config.getDataDir()));
} catch (Exception e) {
- LOGGER.error("meet error while trying to get data size.", e);
+ LOGGER.error("meet error while trying to get data getTotalDataNumber.", e);
return -1;
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/Utils.java b/iotdb/src/main/java/org/apache/iotdb/db/service/Utils.java
index 75826da..6627607 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/Utils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/Utils.java
@@ -34,10 +34,10 @@ public class Utils {
private Utils(){}
/**
- * convert query data set by fetch size.
+ * convert query data set by fetch getTotalDataNumber.
*
* @param queryDataSet -query dataset
- * @param fetchsize -fetch size
+ * @param fetchsize -fetch getTotalDataNumber
* @return -convert query dataset
*/
public static TSQueryDataSet convertQueryDataSetByFetchSize(QueryDataSet queryDataSet,
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/sync/conf/Constans.java b/iotdb/src/main/java/org/apache/iotdb/db/sync/conf/Constans.java
index c810c4c..c1f9856 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/sync/conf/Constans.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/conf/Constans.java
@@ -35,7 +35,7 @@ public class Constans {
public static final String BACK_UP_DIRECTORY_NAME = "backup";
/**
- * Split data file , block size at each transmission
+ * Split data file , block getTotalDataNumber at each transmission
**/
public static final int DATA_CHUNK_SIZE = 64 * 1024 * 1024;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/AuthUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/AuthUtils.java
index 5e253a4..71df1f3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/AuthUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/AuthUtils.java
@@ -47,7 +47,7 @@ public class AuthUtils {
}
/**
- * validate password length.
+ * validate password getTotalDataNumber.
*
* @param password user password
* @throws AuthException Authenticate Exception
@@ -55,7 +55,7 @@ public class AuthUtils {
public static void validatePassword(String password) throws AuthException {
if (password.length() < MIN_PASSWORD_LENGTH) {
throw new AuthException(
- "Password's length must be greater than or equal to " + MIN_USERNAME_LENGTH);
+ "Password's getTotalDataNumber must be greater than or equal to " + MIN_USERNAME_LENGTH);
}
}
@@ -68,7 +68,7 @@ public class AuthUtils {
public static void validateUsername(String username) throws AuthException {
if (username.length() < MIN_USERNAME_LENGTH) {
throw new AuthException(
- "Username's length must be greater than or equal to " + MIN_USERNAME_LENGTH);
+ "Username's getTotalDataNumber must be greater than or equal to " + MIN_USERNAME_LENGTH);
}
}
@@ -81,7 +81,7 @@ public class AuthUtils {
public static void validateRolename(String rolename) throws AuthException {
if (rolename.length() < MIN_ROLENAME_LENGTH) {
throw new AuthException(
- "Role name's length must be greater than or equal to " + MIN_ROLENAME_LENGTH);
+ "Role name's getTotalDataNumber must be greater than or equal to " + MIN_ROLENAME_LENGTH);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
index c276f01..4f17a9c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.utils;
import java.io.File;
/**
- * FileUtils is just used to calculate file attributes like file size, including some measurement
+ * FileUtils is just used to calculate file attributes like file getTotalDataNumber, including some measurement
* conversions among B, KB, MB etc.
*
* @author kangrong
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
index 9d758a4..3927ab1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
@@ -43,7 +43,7 @@ public class MemUtils {
}
/**
- * function for getting the record size.
+ * function for getting the record getTotalDataNumber.
*/
public static long getRecordSize(InsertPlan insertPlan) {
long memSize = 0;
@@ -90,7 +90,7 @@ public class MemUtils {
}
/**
- * function for getting the memory size of the given string.
+ * function for getting the memory getTotalDataNumber of the given string.
*/
public static long getStringMem(String str) {
// wide char (2 bytes each) and 64B String overhead
@@ -98,7 +98,7 @@ public class MemUtils {
}
/**
- * function for getting the memory size of the given data point.
+ * function for getting the memory getTotalDataNumber of the given data point.
*/
public static long getDataPointMem(DataPoint dataPoint) {
// type reference
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java
old mode 100644
new mode 100755
index c488d69..ae942b0
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java
@@ -33,10 +33,10 @@ public class PrimitiveArrayList {
private List<Object> values;
private List<long[]> timestamps;
- private int length; // Total size of all objects of current ArrayList
+ private int length; // Total getTotalDataNumber of all objects of current ArrayList
private int currentIndex; // current index of array
private int currentArrayIndex; // current index of element in current array
- private int currentArraySize; // size of current array
+ private int currentArraySize; // getTotalDataNumber of current array
public PrimitiveArrayList(Class clazz) {
this.clazz = clazz;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListFactory.java
index 8fc154f..b1c949d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.utils;
+import org.apache.iotdb.db.utils.PrimitiveArrayList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2.java
similarity index 61%
copy from iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java
copy to iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2.java
index c488d69..e728cf0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayList.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2.java
@@ -24,79 +24,71 @@ import java.util.List;
import org.apache.iotdb.db.monitor.collector.MemTableWriteTimeCost;
import org.apache.iotdb.db.monitor.collector.MemTableWriteTimeCost.MemTableWriteTimeCostType;
-public class PrimitiveArrayList {
+public class PrimitiveArrayListV2 {
private static final int MAX_SIZE_OF_ONE_ARRAY = 512;
- private static final int INITIAL_SIZE = 512;
+ private static final int INITIAL_SIZE = 16;
private Class clazz;
private List<Object> values;
private List<long[]> timestamps;
- private int length; // Total size of all objects of current ArrayList
- private int currentIndex; // current index of array
- private int currentArrayIndex; // current index of element in current array
- private int currentArraySize; // size of current array
+ private int totalDataNumber; // Total getTotalDataNumber of all objects of current ArrayList
+ private int currentArrayIndex; // current index of array
+ private int offsetInCurrentArray; // current index of element in current array
+ private int currentArraySize; // getTotalDataNumber of current array
- public PrimitiveArrayList(Class clazz) {
+ public PrimitiveArrayListV2(Class clazz) {
this.clazz = clazz;
values = new ArrayList<>();
timestamps = new ArrayList<>();
values.add(Array.newInstance(clazz, INITIAL_SIZE));
timestamps.add(new long[INITIAL_SIZE]);
- length = 0;
+ totalDataNumber = 0;
- currentIndex = 0;
+ currentArrayIndex = 0;
currentArraySize = INITIAL_SIZE;
- currentArrayIndex = -1;
+ offsetInCurrentArray = -1;
}
- private void capacity(int aimSize) {
+ private void checkCapacity(int aimSize) {
if (currentArraySize < aimSize) {
if (currentArraySize < MAX_SIZE_OF_ONE_ARRAY) {
long start = System.currentTimeMillis();
// expand current Array
int newCapacity = Math.min(MAX_SIZE_OF_ONE_ARRAY, currentArraySize * 2);
- values.set(currentIndex,
- expandArray(values.get(currentIndex), currentArraySize, newCapacity));
- timestamps.set(currentIndex,
- (long[]) expandArray(timestamps.get(currentIndex), currentArraySize, newCapacity));
+ values.set(currentArrayIndex,
+ expandArray(values.get(currentArrayIndex), currentArraySize, newCapacity));
+ timestamps.set(currentArrayIndex,
+ (long[]) expandArray(timestamps.get(currentArrayIndex), currentArraySize, newCapacity));
currentArraySize = newCapacity;
MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.CAPACITY_1, start);
} else {
- long start = System.currentTimeMillis();
- // add a new Array to the list
- values.add(Array.newInstance(clazz, INITIAL_SIZE));
- timestamps.add(new long[INITIAL_SIZE]);
- currentIndex++;
- currentArraySize = INITIAL_SIZE;
- currentArrayIndex = -1;
- MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.CAPACITY_2, start);
+ if (currentArrayIndex == values.size() - 1) {
+ // add a new Array to the list
+ values.add(Array.newInstance(clazz, INITIAL_SIZE));
+ timestamps.add(new long[INITIAL_SIZE]);
+ }
+ currentArrayIndex++;
+ currentArraySize = timestamps.get(currentArrayIndex).length;
+ offsetInCurrentArray = -1;
}
}
}
private Object expandArray(Object array, int preLentgh, int aimLength) {
- long start = System.currentTimeMillis();
Class arrayClass = array.getClass().getComponentType();
Object newArray = Array.newInstance(arrayClass, aimLength);
- MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.EXPAND_ARRAY_1, start);
- start = System.currentTimeMillis();
System.arraycopy(array, 0, newArray, 0, preLentgh);
- MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.EXPAND_ARRAY_2, start);
return newArray;
}
public void putTimestamp(long timestamp, Object value) {
- long start = System.currentTimeMillis();
- capacity(currentArrayIndex + 1 + 1);
- MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.PUT_TIMESTAMP_1, start);
- start = System.currentTimeMillis();
- currentArrayIndex++;
- timestamps.get(currentIndex)[currentArrayIndex] = timestamp;
- Array.set(values.get(currentIndex), currentArrayIndex, value);
- length++;
- MemTableWriteTimeCost.getInstance().measure(MemTableWriteTimeCostType.PUT_TIMESTAMP_2, start);
+ checkCapacity(offsetInCurrentArray + 1 + 1);
+ offsetInCurrentArray++;
+ timestamps.get(currentArrayIndex)[offsetInCurrentArray] = timestamp;
+ Array.set(values.get(currentArrayIndex), offsetInCurrentArray, value);
+ totalDataNumber++;
}
public long getTimestamp(int index) {
@@ -113,18 +105,18 @@ public class PrimitiveArrayList {
if (index < 0) {
throw new NegativeArraySizeException("negetive array index:" + index);
}
- if (index >= length) {
+ if (index >= totalDataNumber) {
throw new ArrayIndexOutOfBoundsException("index: " + index);
}
}
- public int size() {
- return length;
+ public int getTotalDataNumber() {
+ return totalDataNumber;
}
@Override
- public PrimitiveArrayList clone() {
- PrimitiveArrayList cloneList = new PrimitiveArrayList(clazz);
+ public PrimitiveArrayListV2 clone() {
+ PrimitiveArrayListV2 cloneList = new PrimitiveArrayListV2(clazz);
cloneList.values.clear();
cloneList.timestamps.clear();
for (Object valueArray : values) {
@@ -133,9 +125,9 @@ public class PrimitiveArrayList {
for (Object timestampArray : timestamps) {
cloneList.timestamps.add((long[]) cloneArray(timestampArray, long.class));
}
- cloneList.length = length;
- cloneList.currentIndex = currentIndex;
+ cloneList.totalDataNumber = totalDataNumber;
cloneList.currentArrayIndex = currentArrayIndex;
+ cloneList.offsetInCurrentArray = offsetInCurrentArray;
cloneList.currentArraySize = currentArraySize;
return cloneList;
}
@@ -146,4 +138,13 @@ public class PrimitiveArrayList {
return cloneArray;
}
+ public Class getClazz() {
+ return clazz;
+ }
+
+ public void reset(){
+ totalDataNumber = 0;
+ currentArrayIndex = 0;
+ offsetInCurrentArray = -1;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveDataListPool.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveDataListPool.java
new file mode 100644
index 0000000..1454857
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/PrimitiveDataListPool.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+/**
+ * Manage all primitive data list in memory, including get and release operation.
+ */
+public class PrimitiveDataListPool {
+
+ private static final Map<Class, LinkedList<PrimitiveArrayListV2>> primitiveArrayListsMap = new HashMap<>();
+
+ static {
+ primitiveArrayListsMap.put(boolean.class, new LinkedList<>());
+ primitiveArrayListsMap.put(int.class, new LinkedList<>());
+ primitiveArrayListsMap.put(long.class, new LinkedList<>());
+ primitiveArrayListsMap.put(float.class, new LinkedList<>());
+ primitiveArrayListsMap.put(double.class, new LinkedList<>());
+ primitiveArrayListsMap.put(Binary.class, new LinkedList<>());
+ }
+
+ private PrimitiveDataListPool() {
+ }
+
+ public synchronized PrimitiveArrayListV2 getPrimitiveDataListByDataType(TSDataType dataType) {
+ switch (dataType) {
+ case BOOLEAN:
+ return getPrimitiveDataList(boolean.class);
+ case INT32:
+ return getPrimitiveDataList(int.class);
+ case INT64:
+ return getPrimitiveDataList(long.class);
+ case FLOAT:
+ return getPrimitiveDataList(float.class);
+ case DOUBLE:
+ return getPrimitiveDataList(double.class);
+ case TEXT:
+ return getPrimitiveDataList(Binary.class);
+ default:
+ throw new UnSupportedDataTypeException("DataType: " + dataType);
+ }
+ }
+
+ private PrimitiveArrayListV2 getPrimitiveDataList(Class clazz) {
+ LinkedList<PrimitiveArrayListV2> primitiveArrayList = primitiveArrayListsMap.get(clazz);
+ return primitiveArrayList.isEmpty() ? new PrimitiveArrayListV2(clazz)
+ : primitiveArrayList.pollFirst();
+ }
+
+ public synchronized void release(PrimitiveArrayListV2 primitiveArrayList) {
+ primitiveArrayList.reset();
+ primitiveArrayListsMap.get(primitiveArrayList.getClazz()).addLast(primitiveArrayList);
+ }
+
+ public static PrimitiveDataListPool getInstance() {
+ return PrimitiveDataListPool.InstanceHolder.INSTANCE;
+ }
+
+ private static class InstanceHolder {
+
+ private InstanceHolder() {
+ }
+
+ private static final PrimitiveDataListPool INSTANCE = new PrimitiveDataListPool();
+ }
+
+ public synchronized int getPrimitiveDataListSizeByDataType(TSDataType dataType){
+ switch (dataType) {
+ case BOOLEAN:
+ return primitiveArrayListsMap.get(boolean.class).size();
+ case INT32:
+ return primitiveArrayListsMap.get(int.class).size();
+ case INT64:
+ return primitiveArrayListsMap.get(long.class).size();
+ case FLOAT:
+ return primitiveArrayListsMap.get(float.class).size();
+ case DOUBLE:
+ return primitiveArrayListsMap.get(double.class).size();
+ case TEXT:
+ return primitiveArrayListsMap.get(Binary.class).size();
+ default:
+ throw new UnSupportedDataTypeException("DataType: " + dataType);
+ }
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java
index d77704e..94131ca 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java
@@ -75,9 +75,9 @@ public abstract class TsPrimitiveType implements Serializable {
}
/**
- * get the size of one instance of current class.
+ * get the getTotalDataNumber of one instance of current class.
*
- * @return size of one instance of current class
+ * @return getTotalDataNumber of one instance of current class
*/
public abstract int getSize();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
index 903d0a4..dbf6a0e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/LogWriter.java
@@ -57,7 +57,7 @@ public class LogWriter implements ILogWriter {
}
logBuffer.flip();
int logSize = logBuffer.limit();
- // 4 bytes length and 8 bytes check sum
+ // 4 bytes getTotalDataNumber and 8 bytes check sum
checkSummer.reset();
checkSummer.update(logBuffer);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/SingleFileLogReader.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/SingleFileLogReader.java
index bacb8f5..752d8c3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/SingleFileLogReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/SingleFileLogReader.java
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
public class SingleFileLogReader implements ILogReader {
private static final Logger logger = LoggerFactory.getLogger(SingleFileLogReader.class);
- public static final int LEAST_LOG_SIZE = 12; // size + checksum
+ public static final int LEAST_LOG_SIZE = 12; // getTotalDataNumber + checksum
private DataInputStream logStream;
private String filepath;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
index 0f60f9c..ca053db 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
@@ -79,12 +79,12 @@ public class FileSizeTest {
isWriteSuccess = false;
e.printStackTrace();
}
- // calculate the delta of data dir file size
+ // calculate the delta of data dir file getTotalDataNumber
dataSizeAfter = FileSize.getInstance().getFileSizesInByte().get(FileSizeConstants.DATA);
long deltaSize = dataSizeAfter - dataSizeBefore;
if (isWriteSuccess) {
- //check if the the delta of data dir file size is equal to the written content size in byte
+ //check if the the delta of data dir file getTotalDataNumber is equal to the written content getTotalDataNumber in byte
assertEquals(contentInBytes.length, deltaSize);
} else {
assertEquals(0, deltaSize);
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/sql/SQLParserTest.java b/iotdb/src/test/java/org/apache/iotdb/db/sql/SQLParserTest.java
index 03a09db..9260f06 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/sql/SQLParserTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/sql/SQLParserTest.java
@@ -388,7 +388,7 @@ public class SQLParserTest {
// recursivePrintSon(astTree, rec);
//
// int i = 0;
- // while (i <= rec.size() - 1) {
+ // while (i <= rec.getTotalDataNumber() - 1) {
// assertEquals(rec.get(i), ans.get(i));
// i++;
// }
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/sync/test/RandomNum.java b/iotdb/src/test/java/org/apache/iotdb/db/sync/test/RandomNum.java
index 1ffee67..6aa0c74 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/sync/test/RandomNum.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/sync/test/RandomNum.java
@@ -54,7 +54,7 @@ public class RandomNum {
/**
* get random text consisting of lowercase letters and numbers.
*
- * @param length -the length of random text
+ * @param length -the getTotalDataNumber of random text
*/
public static String getRandomText(int length) {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2Test.java
similarity index 52%
copy from iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java
copy to iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2Test.java
index 6c8f095..7795af5 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListV2Test.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.utils;
import org.junit.Assert;
import org.junit.Test;
-public class PrimitiveArrayListTest {
+public class PrimitiveArrayListV2Test {
public static void printMemUsed() {
Runtime.getRuntime().gc();
@@ -34,11 +34,11 @@ public class PrimitiveArrayListTest {
}
@Test
- public void test1() {
+ public void testPutAndGet() {
long timestamp = System.currentTimeMillis();
int count = 10000;
- PrimitiveArrayList primitiveArrayList = new PrimitiveArrayList(int.class);
+ PrimitiveArrayListV2 primitiveArrayList = new PrimitiveArrayListV2(int.class);
for (int i = 0; i < count; i++) {
primitiveArrayList.putTimestamp(i, i);
}
@@ -49,6 +49,52 @@ public class PrimitiveArrayListTest {
Assert.assertEquals(i, v);
}
printMemUsed();
- System.out.println("time consume: " + System.currentTimeMillis() + "ms" );
+ System.out.println("time consume: " + (System.currentTimeMillis() - timestamp) + "ms" );
+ }
+
+ @Test
+ public void testReset1() {
+ int count = 10000;
+ PrimitiveArrayListV2 primitiveArrayList = new PrimitiveArrayListV2(int.class);
+ for (int i = 0; i < count; i++) {
+ primitiveArrayList.putTimestamp(i, i);
+ }
+ primitiveArrayList.reset();
+
+ count = 1000;
+ for (int i = 0; i < count; i++) {
+ primitiveArrayList.putTimestamp(i, i + 10000);
+ }
+
+ assert count == primitiveArrayList.getTotalDataNumber();
+ for (int i = 0; i < count; i++) {
+ int v = (int) primitiveArrayList.getValue(i);
+ Assert.assertEquals((long) i, primitiveArrayList.getTimestamp(i));
+ Assert.assertEquals(i + 10000, v);
+ }
+ printMemUsed();
+ }
+
+ @Test
+ public void testReset2() {
+ int count = 1000;
+ PrimitiveArrayListV2 primitiveArrayList = new PrimitiveArrayListV2(int.class);
+ for (int i = 0; i < count; i++) {
+ primitiveArrayList.putTimestamp(i, i);
+ }
+ primitiveArrayList.reset();
+
+ count = 100000;
+ for (int i = 0; i < count; i++) {
+ primitiveArrayList.putTimestamp(i, i + 1000);
+ }
+
+ assert count == primitiveArrayList.getTotalDataNumber();
+ for (int i = 0; i < count; i++) {
+ int v = (int) primitiveArrayList.getValue(i);
+ Assert.assertEquals((long) i, primitiveArrayList.getTimestamp(i));
+ Assert.assertEquals(i + 1000, v);
+ }
+ printMemUsed();
}
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveDataListPoolTest.java
similarity index 59%
rename from iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java
rename to iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveDataListPoolTest.java
index 6c8f095..4996cd1 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveArrayListTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/PrimitiveDataListPoolTest.java
@@ -18,37 +18,39 @@
*/
package org.apache.iotdb.db.utils;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.junit.Assert;
import org.junit.Test;
-public class PrimitiveArrayListTest {
+public class PrimitiveDataListPoolTest {
- public static void printMemUsed() {
- Runtime.getRuntime().gc();
- long size = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- int gb = (int) (size / 1024 / 1024 / 1024);
- int mb = (int) (size / 1024 / 1024 - gb * 1024);
- int kb = (int) (size / 1024 - gb * 1024 * 1024 - mb * 1024);
- int b = (int) (size - gb * 1024 * 1024 * 1024 - mb * 1024 * 1024 - kb * 1024);
- System.out.println("Mem Used:" + gb + "GB, " + mb + "MB, " + kb + "KB, " + b + "B");
+ @Test
+ public void getPrimitiveDataListByDataType() {
}
@Test
- public void test1() {
+ public void release() {
+ int loop = 100;
+ while(loop-- > 0){
+ PrimitiveArrayListV2 primitiveArrayList = PrimitiveDataListPool.getInstance().getPrimitiveDataListByDataType(
+ TSDataType.INT32);
+ testPutAndGet(primitiveArrayList, loop * 100);
+ PrimitiveDataListPool.getInstance().release(primitiveArrayList);
+ assert PrimitiveDataListPool.getInstance().getPrimitiveDataListSizeByDataType(TSDataType.INT32) == 1;
+ }
+ }
- long timestamp = System.currentTimeMillis();
- int count = 10000;
- PrimitiveArrayList primitiveArrayList = new PrimitiveArrayList(int.class);
+ public void testPutAndGet(PrimitiveArrayListV2 primitiveArrayList, int count) {
for (int i = 0; i < count; i++) {
primitiveArrayList.putTimestamp(i, i);
}
+ assert count == primitiveArrayList.getTotalDataNumber();
+
for (int i = 0; i < count; i++) {
int v = (int) primitiveArrayList.getValue(i);
Assert.assertEquals((long) i, primitiveArrayList.getTimestamp(i));
Assert.assertEquals(i, v);
}
- printMemUsed();
- System.out.println("time consume: " + System.currentTimeMillis() + "ms" );
}
-}
+}
\ No newline at end of file
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java
index c6a0905..97686c5 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java
@@ -137,7 +137,7 @@ public class IoTDBLogFileSizeTest {
}
System.out.println(
- "Max size of bufferwrite wal is " + MemUtils.bytesCntToStr(maxLength[0]) + " after "
+ "Max getTotalDataNumber of bufferwrite wal is " + MemUtils.bytesCntToStr(maxLength[0]) + " after "
+ runtime + "ms continuous writing");
}
@@ -197,7 +197,7 @@ public class IoTDBLogFileSizeTest {
}
System.out.println(
- "Max size of overflow wal is " + MemUtils.bytesCntToStr(maxLength[0]) + " after " + runtime
+ "Max getTotalDataNumber of overflow wal is " + MemUtils.bytesCntToStr(maxLength[0]) + " after " + runtime
+ "ms continuous writing");
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
index 1d68e39..848a2bd 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
@@ -103,7 +103,7 @@ public class PerformanceTest {
logNode.forceSync();
System.out.println("forceWalPeriodInMs = " + config.getForceWalPeriodInMs());
System.out.println(
- 3000000 + " logs use " + (System.currentTimeMillis() - time) + " ms at batch size "
+ 3000000 + " logs use " + (System.currentTimeMillis() - time) + " ms at batch getTotalDataNumber "
+ config.getFlushWalThreshold());
logNode.delete();