You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/08/23 06:32:34 UTC
[iotdb] branch cacheoom created (now 198f58a)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch cacheoom
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 198f58a [ISSUE-3805] OOM caused by Chunk cache
This branch includes the following new commits:
new 198f58a [ISSUE-3805] OOM caused by Chunk cache
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [ISSUE-3805] OOM caused by Chunk cache
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch cacheoom
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 198f58a886d03c57b84adf265dba5d02d500ddab
Author: Alima777 <wx...@gmail.com>
AuthorDate: Mon Aug 23 14:31:49 2021 +0800
[ISSUE-3805] OOM caused by Chunk cache
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 62 +++----------
.../db/engine/cache/TimeSeriesMetadataCache.java | 100 ++++++---------------
2 files changed, 40 insertions(+), 122 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 12cd0df..f419a82 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -28,7 +28,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
-import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
@@ -63,55 +62,22 @@ public class ChunkCache {
Caffeine.newBuilder()
.maximumWeight(MEMORY_THRESHOLD_IN_CHUNK_CACHE)
.weigher(
- new Weigher<ChunkMetadata, Chunk>() {
-
- int count = 0;
- int averageSize = 0;
-
- /**
- * The calculation is time consuming, so we won't calculate each entry' size each
- * time. Every 100,000 entry, we will calculate the average size of the first 10
- * entries, and use that to represent the next 99,990 entries' size.
- */
- @Override
- public int weigh(ChunkMetadata chunkMetadata, Chunk chunk) {
- int currentSize;
- if (count < 10) {
- currentSize =
- (int)
- (RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + RamUsageEstimator.sizeOf(chunk));
- averageSize = ((averageSize * count) + currentSize) / (++count);
- entryAverageSize.set(averageSize);
- } else if (count < 100000) {
- count++;
- currentSize = averageSize;
- } else {
- averageSize =
- (int)
- (RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + RamUsageEstimator.sizeOf(chunk));
- count = 1;
- currentSize = averageSize;
- entryAverageSize.set(averageSize);
- }
- return currentSize;
- }
- })
+ (Weigher<ChunkMetadata, Chunk>)
+ (chunkMetadata, chunk) ->
+ (int)
+ (RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + RamUsageEstimator.sizeOf(chunk)))
.recordStats()
.build(
- new CacheLoader<ChunkMetadata, Chunk>() {
- @Override
- public Chunk load(ChunkMetadata chunkMetadata) throws Exception {
- try {
- TsFileSequenceReader reader =
- FileReaderManager.getInstance()
- .get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
- return reader.readMemChunk(chunkMetadata);
- } catch (IOException e) {
- logger.error("Something wrong happened in reading {}", chunkMetadata, e);
- throw e;
- }
+ chunkMetadata -> {
+ try {
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
+ return reader.readMemChunk(chunkMetadata);
+ } catch (IOException e) {
+ logger.error("Something wrong happened in reading {}", chunkMetadata, e);
+ throw e;
}
});
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 1443cbf..e01ccde 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -33,16 +33,23 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
-import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.ref.WeakReference;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -58,7 +65,7 @@ public class TimeSeriesMetadataCache {
config.getAllocateMemoryForTimeSeriesMetaDataCache();
private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
- private final LoadingCache<TimeSeriesMetadataCacheKey, TimeseriesMetadata> lruCache;
+ private final Cache<TimeSeriesMetadataCacheKey, TimeseriesMetadata> lruCache;
private final AtomicLong entryAverageSize = new AtomicLong(0);
@@ -75,77 +82,22 @@ public class TimeSeriesMetadataCache {
Caffeine.newBuilder()
.maximumWeight(MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE)
.weigher(
- new Weigher<TimeSeriesMetadataCacheKey, TimeseriesMetadata>() {
-
- int count = 0;
- int averageSize = 0;
-
- /**
- * The calculation is time consuming, so we won't calculate each entry' size each
- * time. Every 100,000 entry, we will calculate the average size of the first 10
- * entries, and use that to represent the next 99,990 entries' size.
- */
- @Override
- public int weigh(TimeSeriesMetadataCacheKey key, TimeseriesMetadata value) {
- int currentSize;
- if (count < 10) {
- currentSize =
- (int)
- (RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.device)
- + RamUsageEstimator.sizeOf(key.measurement)
- + RamUsageEstimator.shallowSizeOf(value)
- + RamUsageEstimator.sizeOf(value.getMeasurementId())
- + RamUsageEstimator.shallowSizeOf(value.getStatistics())
- + (((ChunkMetadata) value.getChunkMetadataList().get(0))
- .calculateRamSize()
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
- * value.getChunkMetadataList().size()
- + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()));
- averageSize = ((averageSize * count) + currentSize) / (++count);
- entryAverageSize.set(averageSize);
- } else if (count < 100000) {
- count++;
- currentSize = averageSize;
- } else {
- averageSize =
- (int)
- (RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.device)
- + RamUsageEstimator.sizeOf(key.measurement)
- + RamUsageEstimator.shallowSizeOf(value)
- + RamUsageEstimator.sizeOf(value.getMeasurementId())
- + RamUsageEstimator.shallowSizeOf(value.getStatistics())
- + (((ChunkMetadata) value.getChunkMetadataList().get(0))
- .calculateRamSize()
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
- * value.getChunkMetadataList().size()
- + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()));
- count = 1;
- currentSize = averageSize;
- entryAverageSize.set(averageSize);
- }
- return currentSize;
- }
- })
+ (Weigher<TimeSeriesMetadataCacheKey, TimeseriesMetadata>)
+ (key, value) ->
+ (int)
+ (RamUsageEstimator.shallowSizeOf(key)
+ + RamUsageEstimator.sizeOf(key.device)
+ + RamUsageEstimator.sizeOf(key.measurement)
+ + RamUsageEstimator.shallowSizeOf(value)
+ + RamUsageEstimator.sizeOf(value.getMeasurementId())
+ + RamUsageEstimator.shallowSizeOf(value.getStatistics())
+ + (((ChunkMetadata) value.getChunkMetadataList().get(0))
+ .calculateRamSize()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
+ * value.getChunkMetadataList().size()
+ + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList())))
.recordStats()
- .build(
- new CacheLoader<TimeSeriesMetadataCacheKey, TimeseriesMetadata>() {
- @Override
- public TimeseriesMetadata load(TimeSeriesMetadataCacheKey key) throws Exception {
- // bloom filter part
- TsFileSequenceReader reader =
- FileReaderManager.getInstance().get(key.filePath, true);
- BloomFilter bloomFilter = reader.readBloomFilter();
- if (bloomFilter != null
- && !bloomFilter.contains(
- key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
- return null;
- }
- return reader.readTimeseriesMetadata(
- new Path(key.device, key.measurement), false);
- }
- });
+ .build();
}
public static TimeSeriesMetadataCache getInstance() {