You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 07:17:10 UTC
[incubator-iotdb] 01/01: adjust ChunkMetadata size
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch optimize_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit c1268023fa849da93833865aa8008e33434f53ae
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 15:16:53 2020 +0800
adjust ChunkMetadata size
---
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 7 +++++-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 27 ++++++++++++++++------
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
4 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 2069043..5d4d165 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -92,7 +92,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
/**
* Metadata size of per chunk, the default value is 1.5 KB.
*/
- private static final long CHUNK_METADATA_SIZE_IN_BYTE = 1536L;
+ private static long CHUNK_METADATA_SIZE_IN_BYTE = 1536L;
/**
* Average queue length in memtable pool
@@ -259,6 +259,11 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
return totalStorageGroup;
}
+
+ public static void setChunkMetadataSizeInByte(long chunkMetadataSizeInByte) {
+ CHUNK_METADATA_SIZE_IN_BYTE = chunkMetadataSizeInByte;
+ }
+
/**
* Only for test
*/
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 8e28ddc..700e0fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -64,7 +64,7 @@ public class ChunkCache {
long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = (averageSize * count) + currentSize / (++count);
return currentSize;
- } else if (count < 10000) {
+ } else if (count < 100000) {
count++;
return averageSize;
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 0171078..4f20a3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.FileLoaderUtils;
@@ -60,19 +61,31 @@ public class ChunkMetadataCache {
private AtomicLong cacheHitNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- /**
- * approximate estimation of chunkMetaData size
- */
- private long chunkMetaDataSize = 0;
private ChunkMetadataCache(long memoryThreshold) {
lruCache = new LRULinkedHashMap<String, List<ChunkMetadata>>(memoryThreshold, true) {
+ int count = 0;
+ long averageChunkMetadataSize = 0;
+
@Override
protected long calEntrySize(String key, List<ChunkMetadata> value) {
- if (chunkMetaDataSize == 0 && !value.isEmpty()) {
- chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ if (value.isEmpty()) {
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
+ }
+
+ if (count < 10) {
+ long currentSize = RamUsageEstimator.sizeOf(value.get(0));
+ averageChunkMetadataSize = ((averageChunkMetadataSize * count) + currentSize) / (++count);
+ IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageChunkMetadataSize);
+ return key.getBytes().length + currentSize * value.size();
+ } else if (count < 100000) {
+ count++;
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
+ } else {
+ averageChunkMetadataSize = RamUsageEstimator.sizeOf(value.get(0));
+ count = 1;
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
}
- return value.size() * chunkMetaDataSize + key.length() * 2;
}
};
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 3ffc7fa..7fe0ea0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -66,7 +66,7 @@ public class TimeSeriesMetadataCache {
long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
return currentSize;
- } else if (count < 10000) {
+ } else if (count < 100000) {
count++;
return averageSize;
} else {