You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/11/12 13:09:34 UTC
(iotdb) 01/02: Add cache hit situation and actual io size for BloomFilter, TimeSeriesMetadata(including ChunkMetadatList) and Chunk
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryIO
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8e5b9b7463b4582604add685acd823fbf13789a9
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Nov 12 18:34:55 2024 +0800
Add cache hit situation and actual io size for BloomFilter, TimeSeriesMetadata(including ChunkMetadatList) and Chunk
---
.../execution/fragment/QueryStatistics.java | 53 +++++++++
.../execution/operator/source/FileLoaderUtils.java | 9 +-
.../db/storageengine/buffer/BloomFilterCache.java | 80 +++++++++----
.../iotdb/db/storageengine/buffer/ChunkCache.java | 125 ++++++++++++++++-----
.../buffer/TimeSeriesMetadataCache.java | 65 +++++++++--
.../dataregion/read/control/FileReaderManager.java | 32 ++++--
.../read/reader/chunk/DiskAlignedChunkLoader.java | 6 +-
.../read/reader/chunk/DiskChunkLoader.java | 7 +-
.../db/storageengine/buffer/ChunkCacheTest.java | 6 +-
.../commons/exception/IoTDBIORuntimeException.java | 14 +++
pom.xml | 2 +-
11 files changed, 316 insertions(+), 83 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
index b996569a69d..179cc446e00 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
@@ -29,6 +29,11 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class QueryStatistics {
+ private final AtomicLong loadBloomFilterFromCacheCount = new AtomicLong(0);
+ private final AtomicLong loadBloomFilterFromDiskCount = new AtomicLong(0);
+ private final AtomicLong loadBloomFilterActualIOSize = new AtomicLong(0);
+ private final AtomicLong loadBloomFilterTime = new AtomicLong(0);
+
// statistics for count and time of load timeseriesmetadata
private final AtomicLong loadTimeSeriesMetadataDiskSeqCount = new AtomicLong(0);
private final AtomicLong loadTimeSeriesMetadataDiskUnSeqCount = new AtomicLong(0);
@@ -48,6 +53,10 @@ public class QueryStatistics {
private final AtomicLong loadTimeSeriesMetadataAlignedMemSeqTime = new AtomicLong(0);
private final AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqTime = new AtomicLong(0);
+ private final AtomicLong loadTimeSeriesMetadataFromCacheCount = new AtomicLong(0);
+ private final AtomicLong loadTimeSeriesMetadataFromDiskCount = new AtomicLong(0);
+ private final AtomicLong loadTimeSeriesMetadataActualIOSize = new AtomicLong(0);
+
// statistics for count and time of construct chunk readers(disk io and decompress)
private final AtomicLong constructNonAlignedChunkReadersDiskCount = new AtomicLong(0);
private final AtomicLong constructNonAlignedChunkReadersMemCount = new AtomicLong(0);
@@ -59,6 +68,10 @@ public class QueryStatistics {
private final AtomicLong constructAlignedChunkReadersDiskTime = new AtomicLong(0);
private final AtomicLong constructAlignedChunkReadersMemTime = new AtomicLong(0);
+ private final AtomicLong loadChunkFromCacheCount = new AtomicLong(0);
+ private final AtomicLong loadChunkFromDiskCount = new AtomicLong(0);
+ private final AtomicLong loadChunkActualIOSize = new AtomicLong(0);
+
// statistics for count and time of page decode
private final AtomicLong pageReadersDecodeAlignedDiskCount = new AtomicLong(0);
private final AtomicLong pageReadersDecodeAlignedDiskTime = new AtomicLong(0);
@@ -225,6 +238,46 @@ public class QueryStatistics {
return pageReaderMaxUsedMemorySize;
}
+ public AtomicLong getLoadBloomFilterActualIOSize() {
+ return loadBloomFilterActualIOSize;
+ }
+
+ public AtomicLong getLoadBloomFilterFromCacheCount() {
+ return loadBloomFilterFromCacheCount;
+ }
+
+ public AtomicLong getLoadBloomFilterFromDiskCount() {
+ return loadBloomFilterFromDiskCount;
+ }
+
+ public AtomicLong getLoadBloomFilterTime() {
+ return loadBloomFilterTime;
+ }
+
+ public AtomicLong getLoadChunkActualIOSize() {
+ return loadChunkActualIOSize;
+ }
+
+ public AtomicLong getLoadChunkFromCacheCount() {
+ return loadChunkFromCacheCount;
+ }
+
+ public AtomicLong getLoadChunkFromDiskCount() {
+ return loadChunkFromDiskCount;
+ }
+
+ public AtomicLong getLoadTimeSeriesMetadataActualIOSize() {
+ return loadTimeSeriesMetadataActualIOSize;
+ }
+
+ public AtomicLong getLoadTimeSeriesMetadataFromCacheCount() {
+ return loadTimeSeriesMetadataFromCacheCount;
+ }
+
+ public AtomicLong getLoadTimeSeriesMetadataFromDiskCount() {
+ return loadTimeSeriesMetadataFromDiskCount;
+ }
+
public TQueryStatistics toThrift() {
return new TQueryStatistics(
loadTimeSeriesMetadataDiskSeqCount.get(),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
index 1941f3c739f..e940922a96a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
@@ -102,7 +102,8 @@ public class FileLoaderUtils {
seriesPath.getMeasurement()),
allSensors,
resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
- context.isDebug());
+ context.isDebug(),
+ context);
if (timeSeriesMetadata != null) {
long t2 = System.nanoTime();
List<Modification> pathModifications =
@@ -276,7 +277,8 @@ public class FileLoaderUtils {
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
allSensors,
resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
- isDebug);
+ isDebug,
+ context);
if (timeColumn != null) {
// only need time column, like count_time aggregation
if (valueMeasurementList.isEmpty()) {
@@ -304,7 +306,8 @@ public class FileLoaderUtils {
resource.getTsFileID(), deviceId, valueMeasurement),
allSensors,
resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
- isDebug);
+ isDebug,
+ context);
exist = (exist || (valueColumn != null));
valueTimeSeriesMetadataList.add(valueColumn);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
index ca2f3479908..03c56bc4223 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
@@ -19,13 +19,14 @@
package org.apache.iotdb.db.storageengine.buffer;
+import org.apache.iotdb.commons.exception.IoTDBIORuntimeException;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
+import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.utils.BloomFilter;
@@ -36,6 +37,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
/** This class is used to cache <code>BloomFilter</code> in IoTDB. The caching strategy is LRU. */
@SuppressWarnings("squid:S6548")
@@ -49,7 +52,7 @@ public class BloomFilterCache {
private static final boolean CACHE_ENABLE = CONFIG.isMetaDataCacheEnable();
private final AtomicLong entryAverageSize = new AtomicLong(0);
- private final LoadingCache<BloomFilterCacheKey, BloomFilter> lruCache;
+ private final Cache<BloomFilterCacheKey, BloomFilter> lruCache;
private BloomFilterCache() {
if (CACHE_ENABLE) {
@@ -63,35 +66,48 @@ public class BloomFilterCache {
(key, bloomFilter) ->
(int) (key.getRetainedSizeInBytes() + bloomFilter.getRetainedSizeInBytes()))
.recordStats()
- .build(
- key -> {
- TsFileSequenceReader reader =
- FileReaderManager.getInstance().get(key.filePath, true);
- return reader.readBloomFilter();
- });
+ .build();
}
public static BloomFilterCache getInstance() {
return BloomFilterCacheHolder.INSTANCE;
}
+ @TestOnly
public BloomFilter get(BloomFilterCacheKey key) throws IOException {
- return get(key, false);
+ LongConsumer emptyConsumer = l -> {};
+ return get(key, false, emptyConsumer, emptyConsumer, emptyConsumer);
}
- public BloomFilter get(BloomFilterCacheKey key, boolean debug) throws IOException {
- if (!CACHE_ENABLE) {
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
- return reader.readBloomFilter();
- }
+ public BloomFilter get(
+ BloomFilterCacheKey key,
+ boolean debug,
+ LongConsumer ioSizeRecorder,
+ LongConsumer cacheHitAdder,
+ LongConsumer cacheMissAdder)
+ throws IOException {
+ BloomFilterLoader loader = new BloomFilterLoader(ioSizeRecorder);
+ try {
+ if (!CACHE_ENABLE) {
+ return loader.apply(key);
+ }
- BloomFilter bloomFilter = lruCache.get(key);
+ BloomFilter bloomFilter = lruCache.get(key, loader);
- if (debug) {
- DEBUG_LOGGER.info("get bloomFilter from cache where filePath is: {}", key.filePath);
- }
+ if (debug) {
+ DEBUG_LOGGER.info("get bloomFilter from cache where filePath is: {}", key.filePath);
+ }
- return bloomFilter;
+ return bloomFilter;
+ } catch (IoTDBIORuntimeException e) {
+ throw e.getCause();
+ } finally {
+ if (loader.cacheMiss) {
+ cacheMissAdder.accept(1);
+ } else {
+ cacheHitAdder.accept(1);
+ }
+ }
}
public double calculateBloomFilterHitRatio() {
@@ -183,6 +199,32 @@ public class BloomFilterCache {
}
}
+ private static class BloomFilterLoader implements Function<BloomFilterCacheKey, BloomFilter> {
+
+ private boolean cacheMiss = false;
+ private final LongConsumer ioSizeRecorder;
+
+ private BloomFilterLoader(LongConsumer ioSizeRecorder) {
+ this.ioSizeRecorder = ioSizeRecorder;
+ }
+
+ @Override
+ public BloomFilter apply(BloomFilterCacheKey bloomFilterCacheKey) {
+ try {
+ cacheMiss = true;
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance().get(bloomFilterCacheKey.filePath, true, ioSizeRecorder);
+ return reader.readBloomFilter(ioSizeRecorder);
+ } catch (IOException e) {
+ throw new IoTDBIORuntimeException(e);
+ }
+ }
+
+ public boolean isCacheMiss() {
+ return cacheMiss;
+ }
+ }
+
/** singleton pattern. */
private static class BloomFilterCacheHolder {
private static final BloomFilterCache INSTANCE = new BloomFilterCache();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
index 3ad81d39930..719a550d386 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
@@ -19,17 +19,19 @@
package org.apache.iotdb.db.storageengine.buffer;
+import org.apache.iotdb.commons.exception.IoTDBIORuntimeException;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.metric.ChunkCacheMetrics;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.TsFileSequenceReader;
@@ -42,6 +44,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_ALL;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_FILE;
@@ -64,7 +68,7 @@ public class ChunkCache {
SeriesScanCostMetricSet.getInstance();
// to save memory footprint, we don't save measurementId in ChunkHeader of Chunk
- private final LoadingCache<ChunkCacheKey, Chunk> lruCache;
+ private final Cache<ChunkCacheKey, Chunk> lruCache;
private ChunkCache() {
if (CACHE_ENABLE) {
@@ -78,21 +82,7 @@ public class ChunkCache {
(key, chunk) ->
(int) (key.getRetainedSizeInBytes() + chunk.getRetainedSizeInBytes()))
.recordStats()
- .build(
- key -> {
- long startTime = System.nanoTime();
- try {
- TsFileSequenceReader reader =
- FileReaderManager.getInstance().get(key.getFilePath(), key.closed);
- Chunk chunk = reader.readMemChunk(key.offsetOfChunkHeader);
- // to save memory footprint, we don't save measurementId in ChunkHeader of Chunk
- chunk.getHeader().setMeasurementID(null);
- return chunk;
- } finally {
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- READ_CHUNK_FILE, System.nanoTime() - startTime);
- }
- });
+ .build();
// add metrics
MetricService.getInstance().addMetricSet(new ChunkCacheMetrics(this));
@@ -106,36 +96,85 @@ public class ChunkCache {
return ChunkCacheHolder.INSTANCE;
}
+ @TestOnly
+ public Chunk get(
+ ChunkCacheKey chunkCacheKey, List<TimeRange> timeRangeList, Statistics chunkStatistic)
+ throws IOException {
+ LongConsumer emptyConsumer = l -> {};
+ return get(
+ chunkCacheKey,
+ timeRangeList,
+ chunkStatistic,
+ false,
+ emptyConsumer,
+ emptyConsumer,
+ emptyConsumer);
+ }
+
public Chunk get(
ChunkCacheKey chunkCacheKey,
List<TimeRange> timeRangeList,
Statistics chunkStatistic,
- boolean debug)
+ QueryContext queryContext)
+ throws IOException {
+ LongConsumer ioSizeRecorder =
+ queryContext.getQueryStatistics().getLoadChunkActualIOSize()::addAndGet;
+ LongConsumer cacheHitAdder =
+ queryContext.getQueryStatistics().getLoadChunkFromCacheCount()::addAndGet;
+ LongConsumer cacheMissAdder =
+ queryContext.getQueryStatistics().getLoadChunkFromDiskCount()::addAndGet;
+ return get(
+ chunkCacheKey,
+ timeRangeList,
+ chunkStatistic,
+ queryContext.isDebug(),
+ ioSizeRecorder,
+ cacheHitAdder,
+ cacheMissAdder);
+ }
+
+ private Chunk get(
+ ChunkCacheKey chunkCacheKey,
+ List<TimeRange> timeRangeList,
+ Statistics chunkStatistic,
+ boolean debug,
+ LongConsumer ioSizeRecorder,
+ LongConsumer cacheHitAdder,
+ LongConsumer cacheMissAdder)
throws IOException {
long startTime = System.nanoTime();
+ ChunkLoader chunkLoader = new ChunkLoader(ioSizeRecorder);
try {
if (!CACHE_ENABLE) {
- TsFileSequenceReader reader =
- FileReaderManager.getInstance().get(chunkCacheKey.getFilePath(), true);
- Chunk chunk = reader.readMemChunk(chunkCacheKey.offsetOfChunkHeader);
- return new Chunk(
- chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic);
+ Chunk chunk = chunkLoader.apply(chunkCacheKey);
+ return constructChunk(chunk, timeRangeList, chunkStatistic);
}
- Chunk chunk = lruCache.get(chunkCacheKey);
+ Chunk chunk = lruCache.get(chunkCacheKey, chunkLoader);
if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose key is: {}", chunkCacheKey);
}
- return new Chunk(
- chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic);
+ return constructChunk(chunk, timeRangeList, chunkStatistic);
+ } catch (IoTDBIORuntimeException e) {
+ throw e.getCause();
} finally {
+ if (chunkLoader.cacheMiss) {
+ cacheMissAdder.accept(1);
+ } else {
+ cacheHitAdder.accept(1);
+ }
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
READ_CHUNK_ALL, System.nanoTime() - startTime);
}
}
+ private Chunk constructChunk(
+ Chunk chunk, List<TimeRange> timeRangeList, Statistics chunkStatistic) {
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic);
+ }
+
public double calculateChunkHitRatio() {
return lruCache.stats().hitRate();
}
@@ -245,6 +284,40 @@ public class ChunkCache {
}
}
+ private static class ChunkLoader implements Function<ChunkCacheKey, Chunk> {
+
+ private boolean cacheMiss = false;
+ private final LongConsumer ioSizeRecorder;
+
+ private ChunkLoader(LongConsumer ioSizeRecorder) {
+ this.ioSizeRecorder = ioSizeRecorder;
+ }
+
+ @Override
+ public Chunk apply(ChunkCacheKey key) {
+
+ long startTime = System.nanoTime();
+ try {
+
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance().get(key.getFilePath(), key.closed);
+ Chunk chunk = reader.readMemChunk(key.offsetOfChunkHeader);
+ // to save memory footprint, we don't save measurementId in ChunkHeader of Chunk
+ chunk.getHeader().setMeasurementID(null);
+ return chunk;
+ } catch (IOException e) {
+ throw new IoTDBIORuntimeException(e);
+ } finally {
+ SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
+ READ_CHUNK_FILE, System.nanoTime() - startTime);
+ }
+ }
+
+ public boolean isCacheMiss() {
+ return cacheMiss;
+ }
+ }
+
/** singleton pattern. */
private static class ChunkCacheHolder {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 2e7e2b2b627..dfa2e104f04 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
@@ -50,6 +51,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongConsumer;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_CACHE;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_FILE;
@@ -108,9 +110,15 @@ public class TimeSeriesMetadataCache {
TimeSeriesMetadataCacheKey key,
Set<String> allSensors,
boolean ignoreNotExists,
- boolean debug)
+ boolean debug,
+ QueryContext queryContext)
throws IOException {
long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+ queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+ queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
boolean cacheHit = true;
try {
String deviceStringFormat = key.device.toString();
@@ -118,15 +126,21 @@ public class TimeSeriesMetadataCache {
cacheHit = false;
// bloom filter part
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
- BloomFilter bloomFilter = reader.readBloomFilter();
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance().get(filePath, true, bloomFilterIoSizeRecorder);
+ BloomFilter bloomFilter = reader.readBloomFilter(bloomFilterIoSizeRecorder);
+ queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount().incrementAndGet();
if (bloomFilter != null
&& !bloomFilter.contains(
deviceStringFormat + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
+ loadBloomFilterTime = System.nanoTime() - startTime;
return null;
}
+ loadBloomFilterTime = System.nanoTime() - startTime;
+
TimeseriesMetadata timeseriesMetadata =
- reader.readTimeseriesMetadata(key.device, key.measurement, ignoreNotExists);
+ reader.readTimeseriesMetadata(
+ key.device, key.measurement, ignoreNotExists, timeSeriesMetadataIoSizeRecorder);
return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0)
? null
: timeseriesMetadata;
@@ -148,6 +162,7 @@ public class TimeSeriesMetadataCache {
if (timeseriesMetadata == null) {
cacheHit = false;
+ long loadBloomFilterStartTime = System.nanoTime();
// bloom filter part
BloomFilter bloomFilter =
BloomFilterCache.getInstance()
@@ -158,18 +173,33 @@ public class TimeSeriesMetadataCache {
key.timePartitionId,
key.tsFileVersion,
key.compactionVersion),
- debug);
+ debug,
+ bloomFilterIoSizeRecorder,
+ queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()
+ ::addAndGet,
+ queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()
+ ::addAndGet);
if (bloomFilter != null
&& !bloomFilter.contains(
deviceStringFormat + TsFileConstant.PATH_SEPARATOR + key.measurement)) {
if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
+ loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime;
return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
+
+ loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime;
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, true, timeSeriesMetadataIoSizeRecorder);
List<TimeseriesMetadata> timeSeriesMetadataList =
- reader.readTimeseriesMetadata(key.device, key.measurement, allSensors);
+ reader.readTimeseriesMetadata(
+ key.device,
+ key.measurement,
+ allSensors,
+ ignoreNotExists,
+ timeSeriesMetadataIoSizeRecorder);
// put TimeSeriesMetadata of all sensors used in this read into cache
for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
TimeSeriesMetadataCacheKey k =
@@ -207,9 +237,24 @@ public class TimeSeriesMetadataCache {
return new TimeseriesMetadata(timeseriesMetadata);
}
} finally {
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- cacheHit ? READ_TIMESERIES_METADATA_CACHE : READ_TIMESERIES_METADATA_FILE,
- System.nanoTime() - startTime);
+ queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ // in metric panel, loading BloomFilter time is included in loading TimeSeriesMetadata
+ SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
+ READ_TIMESERIES_METADATA_CACHE, System.nanoTime() - startTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ // in metric panel, loading BloomFilter time is included in loading TimeSeriesMetadata
+ SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
+ READ_TIMESERIES_METADATA_FILE, System.nanoTime() - startTime);
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/control/FileReaderManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/control/FileReaderManager.java
index b3189b1ff52..29c4e79550a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/control/FileReaderManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/control/FileReaderManager.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.storageengine.dataregion.read.control;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.UnClosedTsFileReader;
import org.slf4j.Logger;
@@ -33,6 +32,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongConsumer;
/**
* {@link FileReaderManager} is a singleton, which is used to manage all file readers(opened file
@@ -114,6 +114,23 @@ public class FileReaderManager {
@SuppressWarnings("squid:S2095")
public synchronized TsFileSequenceReader get(String filePath, boolean isClosed)
throws IOException {
+ return get(filePath, isClosed, null);
+ }
+
+ /**
+ * Get the reader of the file(tsfile or unseq tsfile) indicated by filePath. If the reader already
+ * exists, just get it from closedFileReaderMap or unclosedFileReaderMap depending on isClosing .
+ * Otherwise a new reader will be created and cached.
+ *
+ * @param filePath the path of the file, of which the reader is desired.
+ * @param isClosed whether the corresponding file still receives insertions or not.
+ * @param ioSizeRecorder can be null
+ * @return the reader of the file specified by filePath.
+ * @throws IOException when reader cannot be created.
+ */
+ @SuppressWarnings("squid:S2095")
+ public synchronized TsFileSequenceReader get(
+ String filePath, boolean isClosed, LongConsumer ioSizeRecorder) throws IOException {
Map<String, TsFileSequenceReader> readerMap =
!isClosed ? unclosedFileReaderMap : closedFileReaderMap;
@@ -127,17 +144,10 @@ public class FileReaderManager {
TsFileSequenceReader tsFileReader = null;
// check if the file is old version
if (!isClosed) {
- tsFileReader = new UnClosedTsFileReader(filePath);
+ tsFileReader = new UnClosedTsFileReader(filePath, ioSizeRecorder);
} else {
- tsFileReader = new TsFileSequenceReader(filePath);
- byte versionNumber = tsFileReader.readVersionNumber();
- if (versionNumber != TSFileConfig.VERSION_NUMBER
- && versionNumber != TSFileConfig.VERSION_NUMBER_V3) {
- tsFileReader.close();
- throw new IOException(
- String.format(
- "The version of TsFile %s is not correct: %s", filePath, versionNumber));
- }
+ // already do the version check in TsFileSequenceReader's constructor
+ tsFileReader = new TsFileSequenceReader(filePath, ioSizeRecorder);
}
readerMap.put(filePath, tsFileReader);
return tsFileReader;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
index d2f2a2da02f..adef246ec8d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
@@ -44,7 +44,6 @@ import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.INI
public class DiskAlignedChunkLoader implements IChunkLoader {
private final QueryContext context;
- private final boolean debug;
private final TsFileResource resource;
@@ -58,7 +57,6 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
public DiskAlignedChunkLoader(
QueryContext context, TsFileResource resource, boolean ignoreAllNullRows) {
this.context = context;
- this.debug = context.isDebug();
this.resource = resource;
this.ignoreAllNullRows = ignoreAllNullRows;
}
@@ -90,7 +88,7 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
resource.isClosed()),
timeChunkMetadata.getDeleteIntervalList(),
timeChunkMetadata.getStatistics(),
- debug);
+ context);
List<Chunk> valueChunkList = new ArrayList<>();
for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) {
valueChunkList.add(
@@ -105,7 +103,7 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
resource.isClosed()),
valueChunkMetadata.getDeleteIntervalList(),
valueChunkMetadata.getStatistics(),
- debug));
+ context));
}
long t2 = System.nanoTime();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
index a956f4ef94f..be33428ae65 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
@@ -44,11 +44,8 @@ public class DiskChunkLoader implements IChunkLoader {
private final TsFileResource resource;
- private final boolean debug;
-
public DiskChunkLoader(QueryContext context, TsFileResource resource) {
this.context = context;
- this.debug = context.isDebug();
this.resource = resource;
}
@@ -63,7 +60,7 @@ public class DiskChunkLoader implements IChunkLoader {
resource.isClosed()),
chunkMetaData.getDeleteIntervalList(),
chunkMetaData.getStatistics(),
- debug);
+ context);
}
@Override
@@ -86,7 +83,7 @@ public class DiskChunkLoader implements IChunkLoader {
resource.isClosed()),
chunkMetaData.getDeleteIntervalList(),
chunkMetaData.getStatistics(),
- debug);
+ context);
long t2 = System.nanoTime();
IChunkReader chunkReader = new ChunkReader(chunk, globalTimeFilter);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
index 1783af0b00f..0477b1bf027 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
@@ -112,8 +112,7 @@ public class ChunkCacheTest {
firstChunkMetadata.getOffsetOfChunkHeader(),
true),
firstChunkMetadata.getDeleteIntervalList(),
- firstChunkMetadata.getStatistics(),
- false);
+ firstChunkMetadata.getStatistics());
ChunkMetadata chunkMetadataKey =
new ChunkMetadata("sensor0", TSDataType.DOUBLE, null, null, 26, new DoubleStatistics());
@@ -128,8 +127,7 @@ public class ChunkCacheTest {
chunkMetadataKey.getOffsetOfChunkHeader(),
true),
chunkMetadataKey.getDeleteIntervalList(),
- chunkMetadataKey.getStatistics(),
- false);
+ chunkMetadataKey.getStatistics());
Assert.assertEquals(chunk1.getHeader(), chunk2.getHeader());
Assert.assertEquals(chunk1.getData(), chunk2.getData());
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/IoTDBIORuntimeException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/IoTDBIORuntimeException.java
new file mode 100644
index 00000000000..b29c709e5da
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/IoTDBIORuntimeException.java
@@ -0,0 +1,14 @@
+package org.apache.iotdb.commons.exception;
+
+import java.io.IOException;
+
+public class IoTDBIORuntimeException extends RuntimeException {
+ public IoTDBIORuntimeException(IOException cause) {
+ super(cause);
+ }
+
+ @Override
+ public synchronized IOException getCause() {
+ return (IOException) super.getCause();
+ }
+}
diff --git a/pom.xml b/pom.xml
index 8738d197a28..90ccc6c0fd2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,7 +166,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>1.2.0-241108-SNAPSHOT</tsfile.version>
+ <tsfile.version>1.2.0-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim