You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/14 07:17:09 UTC
[incubator-iotdb] branch optimize_cache created (now c126802)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch optimize_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at c126802 adjust ChunkMetadata size
This branch includes the following new commits:
new c126802 adjust ChunkMetadata size
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: adjust ChunkMetadata size
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch optimize_cache
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit c1268023fa849da93833865aa8008e33434f53ae
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Apr 14 15:16:53 2020 +0800
adjust ChunkMetadata size
---
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 7 +++++-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 27 ++++++++++++++++------
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
4 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 2069043..5d4d165 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -92,7 +92,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
/**
* Metadata size of per chunk, the default value is 1.5 KB.
*/
- private static final long CHUNK_METADATA_SIZE_IN_BYTE = 1536L;
+ private static long CHUNK_METADATA_SIZE_IN_BYTE = 1536L;
/**
* Average queue length in memtable pool
@@ -259,6 +259,11 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
return totalStorageGroup;
}
+
+ public static void setChunkMetadataSizeInByte(long chunkMetadataSizeInByte) {
+ CHUNK_METADATA_SIZE_IN_BYTE = chunkMetadataSizeInByte;
+ }
+
/**
* Only for test
*/
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 8e28ddc..700e0fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -64,7 +64,7 @@ public class ChunkCache {
long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = (averageSize * count) + currentSize / (++count);
return currentSize;
- } else if (count < 10000) {
+ } else if (count < 100000) {
count++;
return averageSize;
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 0171078..4f20a3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.FileLoaderUtils;
@@ -60,19 +61,31 @@ public class ChunkMetadataCache {
private AtomicLong cacheHitNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- /**
- * approximate estimation of chunkMetaData size
- */
- private long chunkMetaDataSize = 0;
private ChunkMetadataCache(long memoryThreshold) {
lruCache = new LRULinkedHashMap<String, List<ChunkMetadata>>(memoryThreshold, true) {
+ int count = 0;
+ long averageChunkMetadataSize = 0;
+
@Override
protected long calEntrySize(String key, List<ChunkMetadata> value) {
- if (chunkMetaDataSize == 0 && !value.isEmpty()) {
- chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ if (value.isEmpty()) {
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
+ }
+
+ if (count < 10) {
+ long currentSize = RamUsageEstimator.sizeOf(value.get(0));
+ averageChunkMetadataSize = ((averageChunkMetadataSize * count) + currentSize) / (++count);
+ IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageChunkMetadataSize);
+ return key.getBytes().length + currentSize * value.size();
+ } else if (count < 100000) {
+ count++;
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
+ } else {
+ averageChunkMetadataSize = RamUsageEstimator.sizeOf(value.get(0));
+ count = 1;
+ return key.getBytes().length + averageChunkMetadataSize * value.size();
}
- return value.size() * chunkMetaDataSize + key.length() * 2;
}
};
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 3ffc7fa..7fe0ea0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -66,7 +66,7 @@ public class TimeSeriesMetadataCache {
long currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
return currentSize;
- } else if (count < 10000) {
+ } else if (count < 100000) {
count++;
return averageSize;
} else {