You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/06/02 03:13:42 UTC
[iotdb] branch ChunkCacheCaffeine012 created (now 5d588af)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ChunkCacheCaffeine012
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 5d588af fix OOM bug
This branch includes the following new commits:
new 5d588af fix OOM bug
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: fix OOM bug
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ChunkCacheCaffeine012
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d588af36ffdce437c82d743be14d0d8eac64506
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Jun 2 11:09:46 2021 +0800
fix OOM bug
---
server/pom.xml | 5 +
.../db/engine/cache/CacheHitRatioMonitor.java | 16 +-
.../engine/cache/CacheHitRatioMonitorMXBean.java | 8 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 161 ++++++---------
.../db/engine/cache/TimeSeriesMetadataCache.java | 230 +++++++++------------
.../org/apache/iotdb/db/service/TSServiceImpl.java | 9 -
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 17 +-
7 files changed, 186 insertions(+), 260 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index a01a232..e75a1a6 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -197,6 +197,11 @@
<artifactId>netty-buffer</artifactId>
<version>4.1.27.Final</version>
</dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>2.9.1</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
index 7589f54..ccaaaab 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
@@ -58,8 +58,8 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public long getChunkCacheUsedMemory() {
- return ChunkCache.getInstance().getUsedMemory();
+ public long getChunkEvictionCount() {
+ return ChunkCache.getInstance().getEvictionCount();
}
@Override
@@ -68,8 +68,8 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public double getChunkCacheUsedMemoryProportion() {
- return ChunkCache.getInstance().getUsedMemoryProportion();
+ public double getChunkCacheAverageLoadPenalty() {
+ return ChunkCache.getInstance().getAverageLoadPenalty();
}
@Override
@@ -83,8 +83,8 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public long getTimeSeriesMetadataCacheUsedMemory() {
- return TimeSeriesMetadataCache.getInstance().getUsedMemory();
+ public long getTimeSeriesMetadataCacheEvictionCount() {
+ return TimeSeriesMetadataCache.getInstance().getEvictionCount();
}
@Override
@@ -93,8 +93,8 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public double getTimeSeriesCacheUsedMemoryProportion() {
- return TimeSeriesMetadataCache.getInstance().getUsedMemoryProportion();
+ public double getTimeSeriesCacheAverageLoadPenalty() {
+ return TimeSeriesMetadataCache.getInstance().getAverageLoadPenalty();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
index 1c7d261..92de096 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
@@ -22,21 +22,21 @@ public interface CacheHitRatioMonitorMXBean {
double getChunkHitRatio();
- long getChunkCacheUsedMemory();
+ long getChunkEvictionCount();
long getChunkCacheMaxMemory();
- double getChunkCacheUsedMemoryProportion();
+ double getChunkCacheAverageLoadPenalty();
long getChunkCacheAverageSize();
double getTimeSeriesMetadataHitRatio();
- long getTimeSeriesMetadataCacheUsedMemory();
+ long getTimeSeriesMetadataCacheEvictionCount();
long getTimeSeriesMetadataCacheMaxMemory();
- double getTimeSeriesCacheUsedMemoryProportion();
+ double getTimeSeriesCacheAverageLoadPenalty();
long getTimeSeriesMetaDataCacheAverageSize();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 556075f..69de2f2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -28,13 +28,15 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.Weigher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* This class is used to cache <code>Chunk</code> of <code>ChunkMetaData</code> in IoTDB. The
@@ -49,39 +51,64 @@ public class ChunkCache {
config.getAllocateMemoryForChunkCache();
private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
- private final LRULinkedHashMap<ChunkMetadata, Chunk> lruCache;
+ private final LoadingCache<ChunkMetadata, Chunk> lruCache;
- private final AtomicLong cacheHitNum = new AtomicLong();
- private final AtomicLong cacheRequestNum = new AtomicLong();
-
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
+ private final AtomicLong entryAverageSize = new AtomicLong(0);
private ChunkCache() {
if (CACHE_ENABLE) {
logger.info("ChunkCache size = " + MEMORY_THRESHOLD_IN_CHUNK_CACHE);
}
lruCache =
- new LRULinkedHashMap<ChunkMetadata, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE) {
-
- @Override
- protected long calEntrySize(ChunkMetadata key, Chunk value) {
- long currentSize;
- if (count < 10) {
- currentSize =
- RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.sizeOf(value);
- averageSize = ((averageSize * count) + currentSize) / (++count);
- } else if (count < 100000) {
- count++;
- currentSize = averageSize;
- } else {
- averageSize =
- RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.sizeOf(value);
- count = 1;
- currentSize = averageSize;
- }
- return currentSize;
- }
- };
+ Caffeine.newBuilder()
+ .maximumWeight(MEMORY_THRESHOLD_IN_CHUNK_CACHE)
+ .weigher(
+ new Weigher<ChunkMetadata, Chunk>() {
+
+ int count = 0;
+ int averageSize = 0;
+
+ @Override
+ public int weigh(ChunkMetadata chunkMetadata, Chunk chunk) {
+ int currentSize;
+ if (count < 10) {
+ currentSize =
+ (int)
+ (RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + RamUsageEstimator.sizeOf(chunk));
+ averageSize = ((averageSize * count) + currentSize) / (++count);
+ entryAverageSize.set(averageSize);
+ } else if (count < 100000) {
+ count++;
+ currentSize = averageSize;
+ } else {
+ averageSize =
+ (int)
+ (RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + RamUsageEstimator.sizeOf(chunk));
+ count = 1;
+ currentSize = averageSize;
+ entryAverageSize.set(averageSize);
+ }
+ return currentSize;
+ }
+ })
+ .recordStats()
+ .build(
+ new CacheLoader<ChunkMetadata, Chunk>() {
+ @Override
+ public Chunk load(ChunkMetadata chunkMetadata) throws Exception {
+ try {
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
+ return reader.readMemChunk(chunkMetadata);
+ } catch (IOException e) {
+ logger.error("Something wrong happened in reading {}", chunkMetadata, e);
+ throw e;
+ }
+ }
+ });
}
public static ChunkCache getInstance() {
@@ -105,42 +132,12 @@ public class ChunkCache {
chunkMetaData.getStatistics());
}
- cacheRequestNum.incrementAndGet();
-
- Chunk chunk;
- lock.readLock().lock();
- try {
- chunk = lruCache.get(chunkMetaData);
- } finally {
- lock.readLock().unlock();
- }
- if (chunk != null) {
- cacheHitNum.incrementAndGet();
- printCacheLog(true);
- } else {
- printCacheLog(false);
- TsFileSequenceReader reader =
- FileReaderManager.getInstance()
- .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed());
- try {
- chunk = reader.readMemChunk(chunkMetaData);
- } catch (IOException e) {
- logger.error("something wrong happened while reading {}", reader.getFileName());
- throw e;
- }
- lock.writeLock().lock();
- try {
- if (!lruCache.containsKey(chunkMetaData)) {
- lruCache.put(chunkMetaData, chunk);
- }
- } finally {
- lock.writeLock().unlock();
- }
- }
+ Chunk chunk = lruCache.get(chunkMetaData);
if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData);
}
+
return new Chunk(
chunk.getHeader(),
chunk.getData().duplicate(),
@@ -148,61 +145,39 @@ public class ChunkCache {
chunkMetaData.getStatistics());
}
- private void printCacheLog(boolean isHit) {
- if (!logger.isDebugEnabled()) {
- return;
- }
- logger.debug(
- "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.",
- isHit ? "" : "didn't ",
- cacheRequestNum.get(),
- cacheHitNum.get() * 1.0 / cacheRequestNum.get());
- }
-
public double calculateChunkHitRatio() {
- if (cacheRequestNum.get() != 0) {
- return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
- } else {
- return 0;
- }
+ return lruCache.stats().hitRate();
}
- public long getUsedMemory() {
- return lruCache.getUsedMemory();
+ public long getEvictionCount() {
+ return lruCache.stats().evictionCount();
}
public long getMaxMemory() {
- return lruCache.getMaxMemory();
+ return MEMORY_THRESHOLD_IN_CHUNK_CACHE;
}
- public double getUsedMemoryProportion() {
- return lruCache.getUsedMemoryProportion();
+ public double getAverageLoadPenalty() {
+ return lruCache.stats().averageLoadPenalty();
}
public long getAverageSize() {
- return lruCache.getAverageSize();
+ return entryAverageSize.get();
}
/** clear LRUCache. */
public void clear() {
- lock.writeLock().lock();
- if (lruCache != null) {
- lruCache.clear();
- }
- lock.writeLock().unlock();
+ lruCache.invalidateAll();
+ lruCache.cleanUp();
}
public void remove(ChunkMetadata chunkMetaData) {
- lock.writeLock().lock();
- if (chunkMetaData != null) {
- lruCache.remove(chunkMetaData);
- }
- lock.writeLock().unlock();
+ lruCache.invalidate(chunkMetaData);
}
@TestOnly
public boolean isEmpty() {
- return lruCache.isEmpty();
+ return lruCache.estimatedSize() == 0;
}
/** singleton pattern. */
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index acca29a..b21db3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -24,13 +24,17 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.TestOnly;
-import org.apache.iotdb.tsfile.common.cache.Accountable;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.Weigher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,8 +47,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is
@@ -59,12 +61,9 @@ public class TimeSeriesMetadataCache {
config.getAllocateMemoryForTimeSeriesMetaDataCache();
private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
- private final LRULinkedHashMap<TimeSeriesMetadataCacheKey, TimeseriesMetadata> lruCache;
+ private final LoadingCache<TimeSeriesMetadataCacheKey, TimeseriesMetadata> lruCache;
- private final AtomicLong cacheHitNum = new AtomicLong();
- private final AtomicLong cacheRequestNum = new AtomicLong();
-
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
+ private final AtomicLong entryAverageSize = new AtomicLong(0);
private final Map<String, WeakReference<String>> devices =
Collections.synchronizedMap(new WeakHashMap<>());
@@ -76,46 +75,75 @@ public class TimeSeriesMetadataCache {
"TimeseriesMetadataCache size = " + MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE);
}
lruCache =
- new LRULinkedHashMap<TimeSeriesMetadataCacheKey, TimeseriesMetadata>(
- MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE) {
-
- @Override
- protected long calEntrySize(TimeSeriesMetadataCacheKey key, TimeseriesMetadata value) {
- long currentSize;
- if (count < 10) {
- currentSize =
- RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.device)
- + RamUsageEstimator.sizeOf(key.measurement)
- + RamUsageEstimator.shallowSizeOf(value)
- + RamUsageEstimator.sizeOf(value.getMeasurementId())
- + RamUsageEstimator.shallowSizeOf(value.getStatistics())
- + (value.getChunkMetadataList().get(0).calculateRamSize()
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
- * value.getChunkMetadataList().size()
- + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList());
- averageSize = ((averageSize * count) + currentSize) / (++count);
- } else if (count < 100000) {
- count++;
- currentSize = averageSize;
- } else {
- averageSize =
- RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.device)
- + RamUsageEstimator.sizeOf(key.measurement)
- + RamUsageEstimator.shallowSizeOf(value)
- + RamUsageEstimator.sizeOf(value.getMeasurementId())
- + RamUsageEstimator.shallowSizeOf(value.getStatistics())
- + (value.getChunkMetadataList().get(0).calculateRamSize()
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
- * value.getChunkMetadataList().size()
- + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList());
- count = 1;
- currentSize = averageSize;
- }
- return currentSize;
- }
- };
+ Caffeine.newBuilder()
+ .maximumWeight(MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE)
+ .weigher(
+ new Weigher<TimeSeriesMetadataCacheKey, TimeseriesMetadata>() {
+
+ int count = 0;
+ int averageSize = 0;
+
+ @Override
+ public int weigh(TimeSeriesMetadataCacheKey key, TimeseriesMetadata value) {
+ int currentSize;
+ if (count < 10) {
+ currentSize =
+ (int)
+ (RamUsageEstimator.shallowSizeOf(key)
+ + RamUsageEstimator.sizeOf(key.device)
+ + RamUsageEstimator.sizeOf(key.measurement)
+ + RamUsageEstimator.shallowSizeOf(value)
+ + RamUsageEstimator.sizeOf(value.getMeasurementId())
+ + RamUsageEstimator.shallowSizeOf(value.getStatistics())
+ + (((ChunkMetadata) value.getChunkMetadataList().get(0))
+ .calculateRamSize()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
+ * value.getChunkMetadataList().size()
+ + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()));
+ averageSize = ((averageSize * count) + currentSize) / (++count);
+ entryAverageSize.set(averageSize);
+ } else if (count < 100000) {
+ count++;
+ currentSize = averageSize;
+ } else {
+ averageSize =
+ (int)
+ (RamUsageEstimator.shallowSizeOf(key)
+ + RamUsageEstimator.sizeOf(key.device)
+ + RamUsageEstimator.sizeOf(key.measurement)
+ + RamUsageEstimator.shallowSizeOf(value)
+ + RamUsageEstimator.sizeOf(value.getMeasurementId())
+ + RamUsageEstimator.shallowSizeOf(value.getStatistics())
+ + (((ChunkMetadata) value.getChunkMetadataList().get(0))
+ .calculateRamSize()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
+ * value.getChunkMetadataList().size()
+ + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()));
+ count = 1;
+ currentSize = averageSize;
+ entryAverageSize.set(averageSize);
+ }
+ return currentSize;
+ }
+ })
+ .recordStats()
+ .build(
+ new CacheLoader<TimeSeriesMetadataCacheKey, TimeseriesMetadata>() {
+ @Override
+ public TimeseriesMetadata load(TimeSeriesMetadataCacheKey key) throws Exception {
+ // bloom filter part
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance().get(key.filePath, true);
+ BloomFilter bloomFilter = reader.readBloomFilter();
+ if (bloomFilter != null
+ && !bloomFilter.contains(
+ key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
+ return null;
+ }
+ return reader.readTimeseriesMetadata(
+ new Path(key.device, key.measurement), false);
+ }
+ });
}
public static TimeSeriesMetadataCache getInstance() {
@@ -141,20 +169,9 @@ public class TimeSeriesMetadataCache {
return reader.readTimeseriesMetadata(new Path(key.device, key.measurement), false);
}
- cacheRequestNum.incrementAndGet();
-
- TimeseriesMetadata timeseriesMetadata;
- lock.readLock().lock();
- try {
- timeseriesMetadata = lruCache.get(key);
- } finally {
- lock.readLock().unlock();
- }
+ TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
- if (timeseriesMetadata != null) {
- cacheHitNum.incrementAndGet();
- printCacheLog(true);
- } else {
+ if (timeseriesMetadata == null) {
if (debug) {
DEBUG_LOGGER.info(
"Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
@@ -164,16 +181,8 @@ public class TimeSeriesMetadataCache {
synchronized (
devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
// double check
- lock.readLock().lock();
- try {
- timeseriesMetadata = lruCache.get(key);
- } finally {
- lock.readLock().unlock();
- }
- if (timeseriesMetadata != null) {
- cacheHitNum.incrementAndGet();
- printCacheLog(true);
- } else {
+ timeseriesMetadata = lruCache.getIfPresent(key);
+ if (timeseriesMetadata == null) {
Path path = new Path(key.device, key.measurement);
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@ -184,24 +193,17 @@ public class TimeSeriesMetadataCache {
}
return null;
}
- printCacheLog(false);
List<TimeseriesMetadata> timeSeriesMetadataList =
reader.readTimeseriesMetadata(path, allSensors);
// put TimeSeriesMetadata of all sensors used in this query into cache
- lock.writeLock().lock();
- try {
- timeSeriesMetadataList.forEach(
- metadata -> {
- TimeSeriesMetadataCacheKey k =
- new TimeSeriesMetadataCacheKey(
- key.filePath, key.device, metadata.getMeasurementId());
- if (!lruCache.containsKey(k)) {
- lruCache.put(k, metadata);
- }
- });
- timeseriesMetadata = lruCache.get(key);
- } finally {
- lock.writeLock().unlock();
+ for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
+ TimeSeriesMetadataCacheKey k =
+ new TimeSeriesMetadataCacheKey(
+ key.filePath, key.device, metadata.getMeasurementId());
+ lruCache.put(k, metadata);
+ if (metadata.getMeasurementId().equals(key.measurement)) {
+ timeseriesMetadata = metadata;
+ }
}
}
}
@@ -224,64 +226,42 @@ public class TimeSeriesMetadataCache {
}
}
- private void printCacheLog(boolean isHit) {
- if (!logger.isDebugEnabled()) {
- return;
- }
- logger.debug(
- "[TimeSeriesMetadata cache {}hit] The number of requests for cache is {}, hit rate is {}.",
- isHit ? "" : "didn't ",
- cacheRequestNum.get(),
- cacheHitNum.get() * 1.0 / cacheRequestNum.get());
- }
-
public double calculateTimeSeriesMetadataHitRatio() {
- if (cacheRequestNum.get() != 0) {
- return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
- } else {
- return 0;
- }
+ return lruCache.stats().hitRate();
}
- public long getUsedMemory() {
- return lruCache.getUsedMemory();
+ public long getEvictionCount() {
+ return lruCache.stats().evictionCount();
}
public long getMaxMemory() {
- return lruCache.getMaxMemory();
+ return MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE;
}
- public double getUsedMemoryProportion() {
- return lruCache.getUsedMemoryProportion();
+ public double getAverageLoadPenalty() {
+ return lruCache.stats().averageLoadPenalty();
}
public long getAverageSize() {
- return lruCache.getAverageSize();
+ return entryAverageSize.get();
}
/** clear LRUCache. */
public void clear() {
- lock.writeLock().lock();
- if (lruCache != null) {
- lruCache.clear();
- }
- lock.writeLock().unlock();
+ lruCache.invalidateAll();
+ lruCache.cleanUp();
}
public void remove(TimeSeriesMetadataCacheKey key) {
- lock.writeLock().lock();
- if (key != null) {
- lruCache.remove(key);
- }
- lock.writeLock().unlock();
+ lruCache.invalidate(key);
}
@TestOnly
public boolean isEmpty() {
- return lruCache.isEmpty();
+ return lruCache.estimatedSize() == 0;
}
- public static class TimeSeriesMetadataCacheKey implements Accountable {
+ public static class TimeSeriesMetadataCacheKey {
private final String filePath;
private final String device;
@@ -313,16 +293,6 @@ public class TimeSeriesMetadataCache {
public int hashCode() {
return Objects.hash(filePath, device, measurement);
}
-
- @Override
- public void setRamSize(long size) {
- this.ramSize = size;
- }
-
- @Override
- public long getRamSize() {
- return ramSize;
- }
}
/** singleton pattern. */
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 8b6f2ec..aee77cc 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -27,8 +27,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.cost.statistic.Measurement;
import org.apache.iotdb.db.cost.statistic.Operation;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.IoTDBException;
import org.apache.iotdb.db.exception.QueryInBatchStatementException;
@@ -877,13 +875,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (costTime >= config.getSlowQueryThreshold()) {
SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement);
}
- if (plan.isDebug()) {
- SLOW_SQL_LOGGER.info(
- "ChunkCache used memory proportion: {}\n"
- + "TimeSeriesMetadataCache used memory proportion: {}",
- ChunkCache.getInstance().getUsedMemoryProportion(),
- TimeSeriesMetadataCache.getInstance().getUsedMemoryProportion());
- }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 07260bc..de3a563 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -34,7 +33,7 @@ import java.util.List;
import java.util.Objects;
/** Metadata of one chunk. */
-public class ChunkMetadata implements Accountable {
+public class ChunkMetadata {
private String measurementUid;
@@ -63,8 +62,6 @@ public class ChunkMetadata implements Accountable {
private boolean isFromOldTsFile = false;
- private long ramSize;
-
private static final int CHUNK_METADATA_FIXED_RAM_SIZE = 80;
// used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata
@@ -267,20 +264,8 @@ public class ChunkMetadata implements Accountable {
+ Statistics.getSizeByType(dataType);
}
- @Override
- public void setRamSize(long size) {
- this.ramSize = size;
- }
-
- /** must use calculate ram size first */
- @Override
- public long getRamSize() {
- return ramSize;
- }
-
public void mergeChunkMetadata(ChunkMetadata chunkMetadata) {
this.statistics.mergeStatistics(chunkMetadata.getStatistics());
- this.ramSize = calculateRamSize();
}
public void setSeq(boolean seq) {