You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/11/16 09:38:33 UTC
(iotdb) branch master updated: [IOTDB-6237] Improve query back-end execution process
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1dd2f1a3104 [IOTDB-6237] Improve query back-end execution process
1dd2f1a3104 is described below
commit 1dd2f1a3104a6d5d6110c089d359afafcee91b80
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Thu Nov 16 17:38:27 2023 +0800
[IOTDB-6237] Improve query back-end execution process
---
iotdb-core/datanode/pom.xml | 4 +
.../execution/fragment/QueryContext.java | 23 ++-
.../execution/operator/process/SortOperator.java | 4 +-
.../execution/operator/process/TopKOperator.java | 4 +-
.../schemaregion/utils/ResourceByPathUtils.java | 10 +-
.../db/storageengine/buffer/BloomFilterCache.java | 56 ++++----
.../storageengine/buffer/CacheHitRatioMonitor.java | 3 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 155 +++++++++++++++------
.../buffer/TimeSeriesMetadataCache.java | 125 +++++++++++------
.../db/storageengine/dataregion/DataRegion.java | 2 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 3 -
.../read/reader/chunk/DiskAlignedChunkLoader.java | 26 +++-
.../read/reader/chunk/DiskChunkLoader.java | 27 +++-
.../metadata/DiskAlignedChunkMetadataLoader.java | 4 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 4 +-
.../metadata/MemAlignedChunkMetadataLoader.java | 4 +-
.../chunk/metadata/MemChunkMetadataLoader.java | 4 +-
.../storageengine/dataregion/tsfile/TsFileID.java | 96 +++++++++++++
.../dataregion/tsfile/TsFileManager.java | 10 +-
.../dataregion/tsfile/TsFileResource.java | 104 +++-----------
.../TsFileOverlapValidationAndRepairTool.java | 3 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 16 ++-
.../execution/operator/AlignedSeriesTestUtil.java | 5 +-
.../execution/operator/SortOperatorTest.java | 47 ++++---
.../execution/operator/TopKOperatorTest.java | 4 +
.../storageengine/buffer/BloomFilterCacheTest.java | 40 +++++-
.../db/storageengine/buffer/ChunkCacheTest.java | 65 ++++-----
.../buffer/TimeSeriesMetadataCacheTest.java | 28 ----
.../compaction/AbstractCompactionTest.java | 3 +
.../compaction/CompactionSchedulerTest.java | 7 +-
.../CompactionSchedulerWithFastPerformerTest.java | 5 +
.../cross/CrossSpaceCompactionExceptionTest.java | 4 +
.../cross/InsertionCrossSpaceCompactionTest.java | 8 +-
.../compaction/inner/InnerCompactionTest.java | 2 +
.../InnerSeqCompactionWithFastPerformerTest.java | 8 +-
...nerSeqCompactionWithReadChunkPerformerTest.java | 2 +
.../InnerUnseqCompactionWithFastPerformerTest.java | 2 +
...rUnseqCompactionWithReadPointPerformerTest.java | 2 +
.../utils/CompactionFileGeneratorUtils.java | 35 +----
.../dataregion/memtable/TsFileProcessorTest.java | 7 +-
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 6 +
.../read/reader/series/SeriesReaderTestUtil.java | 2 +
.../rescon/memory/ResourceManagerTest.java | 2 +
.../apache/iotdb/db/utils/sort/SortUtilTest.java | 9 ++
.../apache/iotdb/commons/conf/IoTDBConstant.java | 6 -
.../iotdb/tsfile/file/header/ChunkHeader.java | 50 +++++--
.../tsfile/file/metadata/AlignedChunkMetadata.java | 10 --
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 112 +++++++--------
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 2 -
.../tsfile/file/metadata/TimeseriesMetadata.java | 55 +++++---
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 31 -----
.../file/metadata/statistics/BinaryStatistics.java | 22 ++-
.../metadata/statistics/BooleanStatistics.java | 17 ++-
.../file/metadata/statistics/DoubleStatistics.java | 17 ++-
.../file/metadata/statistics/FloatStatistics.java | 17 ++-
.../metadata/statistics/IntegerStatistics.java | 17 ++-
.../file/metadata/statistics/LongStatistics.java | 17 ++-
.../file/metadata/statistics/Statistics.java | 18 +--
.../file/metadata/statistics/TimeStatistics.java | 16 ++-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 30 ++--
.../org/apache/iotdb/tsfile/read/common/Chunk.java | 28 +++-
.../org/apache/iotdb/tsfile/utils/BloomFilter.java | 95 ++++---------
.../apache/iotdb/tsfile/utils/FilePathUtils.java | 38 -----
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 5 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +-
.../tsfile/file/metadata/utils/TestHelper.java | 1 -
66 files changed, 891 insertions(+), 702 deletions(-)
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index c86aec5e85e..9c1e227c44a 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -199,6 +199,10 @@
<groupId>org.openjdk.jol</groupId>
<artifactId>jol-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>slice</artifactId>
+ </dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
index 3b9b6046574..b22f2b5f3cc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory.ModsSerializer;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -31,8 +33,10 @@ import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/** QueryContext contains the shared information with in a query. */
public class QueryContext {
@@ -56,14 +60,16 @@ public class QueryContext {
private volatile boolean isInterrupted = false;
+ private final Set<TsFileID> nonExistentModFiles = new HashSet<>();
+
public QueryContext() {}
public QueryContext(long queryId) {
- this(queryId, false, System.currentTimeMillis(), "", 0);
+ this(queryId, false, System.currentTimeMillis(), 0);
}
/** Every time we generate the queryContext, register it to queryTimeManager. */
- public QueryContext(long queryId, boolean debug, long startTime, String statement, long timeout) {
+ public QueryContext(long queryId, boolean debug, long startTime, long timeout) {
this.queryId = queryId;
this.debug = debug;
this.startTime = startTime;
@@ -74,9 +80,15 @@ public class QueryContext {
* Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
*/
- public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
+ public List<Modification> getPathModifications(TsFileResource tsFileResource, PartialPath path) {
// if the mods file does not exist, do not add it to the cache
+ if (nonExistentModFiles.contains(tsFileResource.getTsFileID())) {
+ return Collections.emptyList();
+ }
+
+ ModificationFile modFile = tsFileResource.getModFile();
if (!modFile.exists()) {
+ nonExistentModFiles.add(tsFileResource.getTsFileID());
return Collections.emptyList();
}
@@ -96,11 +108,12 @@ public class QueryContext {
* Find the modifications of all aligned 'paths' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
*/
- public List<List<Modification>> getPathModifications(ModificationFile modFile, AlignedPath path) {
+ public List<List<Modification>> getPathModifications(
+ TsFileResource tsFileResource, AlignedPath path) {
int n = path.getMeasurementList().size();
List<List<Modification>> ans = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
- ans.add(getPathModifications(modFile, path.getPathWithMeasurement(i)));
+ ans.add(getPathModifications(tsFileResource, path.getPathWithMeasurement(i)));
}
return ans;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
index be2baaa78ed..78bf446aae0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
@@ -72,7 +72,7 @@ public class SortOperator implements ProcessOperator {
private static final Logger logger = LoggerFactory.getLogger(SortOperator.class);
- private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES =
+ private final int maxReturnSize =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();
public SortOperator(
@@ -337,7 +337,7 @@ public class SortOperator implements ProcessOperator {
@Override
public long calculateMaxReturnSize() {
- return DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ return maxReturnSize;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
index 9c8a9cef3e2..547f1edebdc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
@@ -290,7 +290,9 @@ public class TopKOperator implements ProcessOperator {
tsBlockBuilder.reset();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- for (int i = resultReturnSize; i < topKResult.length; i++) {
+ for (int i = resultReturnSize, size = (topKResult == null ? 0 : topKResult.length);
+ i < size;
+ i++) {
MergeSortKey mergeSortKey = topKResult[i];
TsBlock targetBlock = mergeSortKey.tsBlock;
tsBlockBuilder
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
index b2d5e277bdf..9beb4223a7f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
@@ -34,7 +34,6 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGr
import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
-import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -119,7 +118,6 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils {
public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList) {
TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata();
- timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1);
timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
timeTimeSeriesMetadata.setMeasurementId("");
timeTimeSeriesMetadata.setTsDataType(TSDataType.VECTOR);
@@ -131,7 +129,6 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils {
List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>();
for (IMeasurementSchema valueChunkMetadata : (partialPath.getSchemaList())) {
TimeseriesMetadata valueMetadata = new TimeseriesMetadata();
- valueMetadata.setOffsetOfChunkMetaDataList(-1);
valueMetadata.setDataSizeOfChunkMetaDataList(-1);
valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementId());
valueMetadata.setTsDataType(valueChunkMetadata.getType());
@@ -265,9 +262,8 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils {
@Override
public List<IChunkMetadata> getVisibleMetadataListFromWriter(
RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) {
- ModificationFile modificationFile = tsFileResource.getModFile();
List<List<Modification>> modifications =
- context.getPathModifications(modificationFile, partialPath);
+ context.getPathModifications(tsFileResource, partialPath);
List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
List<ChunkMetadata> timeChunkMetadataList =
@@ -326,7 +322,6 @@ class MeasurementResourceByPathUtils extends ResourceByPathUtils {
TimeseriesMetadata timeSeriesMetadata = new TimeseriesMetadata();
timeSeriesMetadata.setMeasurementId(partialPath.getMeasurementSchema().getMeasurementId());
timeSeriesMetadata.setTsDataType(partialPath.getMeasurementSchema().getType());
- timeSeriesMetadata.setOffsetOfChunkMetaDataList(-1);
timeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
Statistics<? extends Serializable> seriesStatistics =
@@ -415,8 +410,7 @@ class MeasurementResourceByPathUtils extends ResourceByPathUtils {
@Override
public List<IChunkMetadata> getVisibleMetadataListFromWriter(
RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) {
- ModificationFile modificationFile = tsFileResource.getModFile();
- List<Modification> modifications = context.getPathModifications(modificationFile, partialPath);
+ List<Modification> modifications = context.getPathModifications(tsFileResource, partialPath);
List<IChunkMetadata> chunkMetadataList =
new ArrayList<>(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
index c2c0483f552..f138a1808f8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
@@ -25,13 +25,11 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.BloomFilter;
-import org.apache.iotdb.tsfile.utils.FilePathUtils;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
+import org.openjdk.jol.info.ClassLayout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,19 +41,19 @@ import java.util.concurrent.atomic.AtomicLong;
@SuppressWarnings("squid:S6548")
public class BloomFilterCache {
- private static final Logger logger = LoggerFactory.getLogger(BloomFilterCache.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(BloomFilterCache.class);
private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
- private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
private static final long MEMORY_THRESHOLD_IN_BLOOM_FILTER_CACHE =
- config.getAllocateMemoryForBloomFilterCache();
- private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
+ CONFIG.getAllocateMemoryForBloomFilterCache();
+ private static final boolean CACHE_ENABLE = CONFIG.isMetaDataCacheEnable();
private final AtomicLong entryAverageSize = new AtomicLong(0);
private final LoadingCache<BloomFilterCacheKey, BloomFilter> lruCache;
private BloomFilterCache() {
if (CACHE_ENABLE) {
- logger.info("BloomFilterCache size = {}", MEMORY_THRESHOLD_IN_BLOOM_FILTER_CACHE);
+ LOGGER.info("BloomFilterCache size = {}", MEMORY_THRESHOLD_IN_BLOOM_FILTER_CACHE);
}
lruCache =
Caffeine.newBuilder()
@@ -63,10 +61,7 @@ public class BloomFilterCache {
.weigher(
(Weigher<BloomFilterCacheKey, BloomFilter>)
(key, bloomFilter) ->
- (int)
- (RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.tsFilePrefixPath)
- + RamUsageEstimator.sizeOf(bloomFilter)))
+ (int) (key.getRetainedSizeInBytes() + bloomFilter.getRetainedSizeInBytes()))
.recordStats()
.build(
key -> {
@@ -137,22 +132,30 @@ public class BloomFilterCache {
public static class BloomFilterCacheKey {
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(BloomFilterCacheKey.class).instanceSize();
+
// There is no need to add this field size while calculating the size of BloomFilterCacheKey,
// because filePath is get from TsFileResource, different BloomFilterCacheKey of the same file
// share this String.
private final String filePath;
- private final String tsFilePrefixPath;
+ private final int regionId;
+ private final long timePartitionId;
private final long tsFileVersion;
// high 32 bit is compaction level, low 32 bit is merge count
private final long compactionVersion;
- public BloomFilterCacheKey(String filePath) {
+ public BloomFilterCacheKey(
+ String filePath,
+ int regionId,
+ long timePartitionId,
+ long tsFileVersion,
+ long compactionVersion) {
this.filePath = filePath;
- Pair<String, long[]> tsFilePrefixPathAndTsFileVersionPair =
- FilePathUtils.getTsFilePrefixPathAndTsFileVersionPair(filePath);
- this.tsFilePrefixPath = tsFilePrefixPathAndTsFileVersionPair.left;
- this.tsFileVersion = tsFilePrefixPathAndTsFileVersionPair.right[0];
- this.compactionVersion = tsFilePrefixPathAndTsFileVersionPair.right[1];
+ this.regionId = regionId;
+ this.timePartitionId = timePartitionId;
+ this.tsFileVersion = tsFileVersion;
+ this.compactionVersion = compactionVersion;
}
@Override
@@ -163,15 +166,20 @@ public class BloomFilterCache {
if (o == null || getClass() != o.getClass()) {
return false;
}
- BloomFilterCache.BloomFilterCacheKey that = (BloomFilterCache.BloomFilterCacheKey) o;
- return tsFileVersion == that.tsFileVersion
- && compactionVersion == that.compactionVersion
- && tsFilePrefixPath.equals(that.tsFilePrefixPath);
+ BloomFilterCacheKey that = (BloomFilterCacheKey) o;
+ return regionId == that.regionId
+ && timePartitionId == that.timePartitionId
+ && tsFileVersion == that.tsFileVersion
+ && compactionVersion == that.compactionVersion;
}
@Override
public int hashCode() {
- return Objects.hash(tsFilePrefixPath, tsFileVersion, compactionVersion);
+ return Objects.hash(regionId, timePartitionId, tsFileVersion, compactionVersion);
+ }
+
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/CacheHitRatioMonitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/CacheHitRatioMonitor.java
index 3bb5d94a6b1..0d610c5af2f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/CacheHitRatioMonitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/CacheHitRatioMonitor.java
@@ -79,7 +79,8 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
@Override
public long getChunkCacheAverageSize() {
- return ChunkCache.getInstance().getAverageSize();
+ // doesn't implement
+ return 0;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
index 3c871f51247..6fe3509994d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
@@ -26,19 +26,22 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.metric.ChunkCacheMetrics;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
+import org.openjdk.jol.info.ClassLayout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.List;
+import java.util.Objects;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_ALL;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_FILE;
@@ -50,41 +53,41 @@ import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.REA
@SuppressWarnings("squid:S6548")
public class ChunkCache {
- private static final Logger logger = LoggerFactory.getLogger(ChunkCache.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(ChunkCache.class);
private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
- private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE =
- config.getAllocateMemoryForChunkCache();
- private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
+ CONFIG.getAllocateMemoryForChunkCache();
+ private static final boolean CACHE_ENABLE = CONFIG.isMetaDataCacheEnable();
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
- private final LoadingCache<ChunkMetadata, Chunk> lruCache;
- private final AtomicLong entryAverageSize = new AtomicLong(0);
+ // to save memory footprint, we don't save measurementId in ChunkHeader of Chunk
+ private final LoadingCache<ChunkCacheKey, Chunk> lruCache;
private ChunkCache() {
if (CACHE_ENABLE) {
- logger.info("ChunkCache size = {}", MEMORY_THRESHOLD_IN_CHUNK_CACHE);
+ LOGGER.info("ChunkCache size = {}", MEMORY_THRESHOLD_IN_CHUNK_CACHE);
}
lruCache =
Caffeine.newBuilder()
.maximumWeight(MEMORY_THRESHOLD_IN_CHUNK_CACHE)
.weigher(
- (Weigher<ChunkMetadata, Chunk>)
- (chunkMetadata, chunk) ->
- (int)
- (RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + RamUsageEstimator.sizeOf(chunk)))
+ (Weigher<ChunkCacheKey, Chunk>)
+ (key, chunk) ->
+ (int) (key.getRetainedSizeInBytes() + chunk.getRetainedSizeInBytes()))
.recordStats()
.build(
- chunkMetadata -> {
+ key -> {
long startTime = System.nanoTime();
try {
TsFileSequenceReader reader =
- FileReaderManager.getInstance()
- .get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
- return reader.readMemChunk(chunkMetadata);
+ FileReaderManager.getInstance().get(key.getFilePath(), true);
+ Chunk chunk = reader.readMemChunk(key.offsetOfChunkHeader);
+ // to save memory footprint, we don't save measurementId in ChunkHeader of Chunk
+ chunk.getHeader().setMeasurementID(null);
+ return chunk;
} finally {
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
READ_CHUNK_FILE, System.nanoTime() - startTime);
@@ -103,36 +106,30 @@ public class ChunkCache {
return ChunkCacheHolder.INSTANCE;
}
- public Chunk get(ChunkMetadata chunkMetaData) throws IOException {
- return get(chunkMetaData, false);
- }
-
- public Chunk get(ChunkMetadata chunkMetaData, boolean debug) throws IOException {
+ public Chunk get(
+ ChunkCacheKey chunkCacheKey,
+ List<TimeRange> timeRangeList,
+ Statistics chunkStatistic,
+ boolean debug)
+ throws IOException {
long startTime = System.nanoTime();
try {
if (!CACHE_ENABLE) {
TsFileSequenceReader reader =
- FileReaderManager.getInstance()
- .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed());
- Chunk chunk = reader.readMemChunk(chunkMetaData);
+ FileReaderManager.getInstance().get(chunkCacheKey.getFilePath(), true);
+ Chunk chunk = reader.readMemChunk(chunkCacheKey.offsetOfChunkHeader);
return new Chunk(
- chunk.getHeader(),
- chunk.getData().duplicate(),
- chunkMetaData.getDeleteIntervalList(),
- chunkMetaData.getStatistics());
+ chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic);
}
- Chunk chunk = lruCache.get(chunkMetaData);
+ Chunk chunk = lruCache.get(chunkCacheKey);
if (debug) {
- DEBUG_LOGGER.info("get chunk from cache whose meta data is: {}", chunkMetaData);
+ DEBUG_LOGGER.info("get chunk from cache whose key is: {}", chunkCacheKey);
}
return new Chunk(
- chunk.getHeader(),
- chunk.getData().duplicate(),
- chunkMetaData.getDeleteIntervalList(),
- chunkMetaData.getStatistics());
+ chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic);
} finally {
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
READ_CHUNK_ALL, System.nanoTime() - startTime);
@@ -155,25 +152,93 @@ public class ChunkCache {
return lruCache.stats().averageLoadPenalty();
}
- public long getAverageSize() {
- return entryAverageSize.get();
- }
-
/** clear LRUCache. */
public void clear() {
lruCache.invalidateAll();
lruCache.cleanUp();
}
- public void remove(ChunkMetadata chunkMetaData) {
- lruCache.invalidate(chunkMetaData);
- }
-
@TestOnly
public boolean isEmpty() {
return lruCache.asMap().isEmpty();
}
+ public static class ChunkCacheKey {
+
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(ChunkCacheKey.class).instanceSize();
+
+ // There is no need to add this field size while calculating the size of ChunkCacheKey,
+ // because filePath is get from TsFileResource, different ChunkCacheKey of the same file
+ // share this String.
+ private final String filePath;
+ private final int regionId;
+ private final long timePartitionId;
+ private final long tsFileVersion;
+ // high 32 bit is compaction level, low 32 bit is merge count
+ private final long compactionVersion;
+
+ private final long offsetOfChunkHeader;
+
+ public ChunkCacheKey(String filePath, TsFileID tsfileId, long offsetOfChunkHeader) {
+ this.filePath = filePath;
+ this.regionId = tsfileId.regionId;
+ this.timePartitionId = tsfileId.timePartitionId;
+ this.tsFileVersion = tsfileId.fileVersion;
+ this.compactionVersion = tsfileId.compactionVersion;
+ this.offsetOfChunkHeader = offsetOfChunkHeader;
+ }
+
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ChunkCacheKey that = (ChunkCacheKey) o;
+ return regionId == that.regionId
+ && timePartitionId == that.timePartitionId
+ && tsFileVersion == that.tsFileVersion
+ && compactionVersion == that.compactionVersion
+ && offsetOfChunkHeader == that.offsetOfChunkHeader;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ regionId, timePartitionId, tsFileVersion, compactionVersion, offsetOfChunkHeader);
+ }
+
+ @Override
+ public String toString() {
+ return "ChunkCacheKey{"
+ + "filePath='"
+ + filePath
+ + '\''
+ + ", regionId="
+ + regionId
+ + ", timePartitionId="
+ + timePartitionId
+ + ", tsFileVersion="
+ + tsFileVersion
+ + ", compactionVersion="
+ + compactionVersion
+ + ", offsetOfChunkHeader="
+ + offsetOfChunkHeader
+ + '}';
+ }
+ }
+
/** singleton pattern. */
private static class ChunkCacheHolder {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 5092b19fc52..eea187e694f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@ -27,18 +27,16 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.BloomFilter;
-import org.apache.iotdb.tsfile.utils.FilePathUtils;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.Weigher;
+import org.openjdk.jol.info.ClassLayout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +50,7 @@ import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import static io.airlift.slice.SizeOf.sizeOfCharArray;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_CACHE;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_FILE;
@@ -83,7 +82,7 @@ public class TimeSeriesMetadataCache {
private TimeSeriesMetadataCache() {
if (CACHE_ENABLE) {
logger.info(
- "TimeseriesMetadataCache size = {}", MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE);
+ "TimeSeriesMetadataCache size = {}", MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE);
}
lruCache =
Caffeine.newBuilder()
@@ -91,22 +90,7 @@ public class TimeSeriesMetadataCache {
.weigher(
(Weigher<TimeSeriesMetadataCacheKey, TimeseriesMetadata>)
(key, value) ->
- (int)
- (RamUsageEstimator.shallowSizeOf(key)
- + RamUsageEstimator.sizeOf(key.device)
- + RamUsageEstimator.sizeOf(key.measurement)
- + RamUsageEstimator.sizeOf(key.tsFilePrefixPath)
- + RamUsageEstimator.sizeOf(key.tsFileVersion)
- + RamUsageEstimator.shallowSizeOf(value)
- + RamUsageEstimator.sizeOf(value.getMeasurementId())
- + RamUsageEstimator.shallowSizeOf(value.getStatistics())
- + (value.getChunkMetadataList().get(0) == null
- ? 0
- : ((ChunkMetadata) value.getChunkMetadataList().get(0))
- .calculateRamSize()
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
- * value.getChunkMetadataList().size()
- + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList())))
+ (int) (key.getRetainedSizeInBytes() + value.getRetainedSizeInBytes()))
.recordStats()
.build();
// add metrics
@@ -119,6 +103,7 @@ public class TimeSeriesMetadataCache {
@SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning
public TimeseriesMetadata get(
+ String filePath,
TimeSeriesMetadataCacheKey key,
Set<String> allSensors,
boolean ignoreNotExists,
@@ -131,7 +116,7 @@ public class TimeSeriesMetadataCache {
cacheHit = false;
// bloom filter part
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null
&& !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
@@ -148,13 +133,12 @@ public class TimeSeriesMetadataCache {
if (timeseriesMetadata == null) {
if (debug) {
- DEBUG_LOGGER.info(
- "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
+ DEBUG_LOGGER.info("Cache miss: {}.{} in file: {}", key.device, key.measurement, filePath);
DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
}
// allow for the parallelism of different devices
synchronized (
- devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
+ devices.computeIfAbsent(key.device + SEPARATOR + filePath, WeakReference::new)) {
// double check
timeseriesMetadata = lruCache.getIfPresent(key);
if (timeseriesMetadata == null) {
@@ -163,7 +147,14 @@ public class TimeSeriesMetadataCache {
// bloom filter part
BloomFilter bloomFilter =
BloomFilterCache.getInstance()
- .get(new BloomFilterCache.BloomFilterCacheKey(key.filePath), debug);
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(
+ filePath,
+ key.regionId,
+ key.timePartitionId,
+ key.tsFileVersion,
+ key.compactionVersion),
+ debug);
if (bloomFilter != null
&& !bloomFilter.contains(
key.device + TsFileConstant.PATH_SEPARATOR + key.measurement)) {
@@ -172,14 +163,19 @@ public class TimeSeriesMetadataCache {
}
return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
List<TimeseriesMetadata> timeSeriesMetadataList =
reader.readTimeseriesMetadata(key.device, key.measurement, allSensors);
// put TimeSeriesMetadata of all sensors used in this read into cache
for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
TimeSeriesMetadataCacheKey k =
new TimeSeriesMetadataCacheKey(
- key.filePath, key.device, metadata.getMeasurementId());
+ key.regionId,
+ key.timePartitionId,
+ key.tsFileVersion,
+ key.compactionVersion,
+ key.device,
+ metadata.getMeasurementId());
if (metadata.getStatistics().getCount() != 0) {
lruCache.put(k, metadata);
}
@@ -201,7 +197,7 @@ public class TimeSeriesMetadataCache {
"Get timeseries: {}.{} metadata in file: {} from cache: {}.",
key.device,
key.measurement,
- key.filePath,
+ filePath,
timeseriesMetadata);
}
return new TimeseriesMetadata(timeseriesMetadata);
@@ -254,25 +250,48 @@ public class TimeSeriesMetadataCache {
public static class TimeSeriesMetadataCacheKey {
- private final String filePath;
- private final String tsFilePrefixPath;
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(TimeSeriesMetadataCacheKey.class).instanceSize()
+ + 2 * ClassLayout.parseClass(String.class).instanceSize();
+
+ private final int regionId;
+ private final long timePartitionId;
private final long tsFileVersion;
// high 32 bit is compaction level, low 32 bit is merge count
private final long compactionVersion;
private final String device;
private final String measurement;
- public TimeSeriesMetadataCacheKey(String filePath, String device, String measurement) {
- this.filePath = filePath;
- Pair<String, long[]> tsFilePrefixPathAndTsFileVersionPair =
- FilePathUtils.getTsFilePrefixPathAndTsFileVersionPair(filePath);
- this.tsFilePrefixPath = tsFilePrefixPathAndTsFileVersionPair.left;
- this.tsFileVersion = tsFilePrefixPathAndTsFileVersionPair.right[0];
- this.compactionVersion = tsFilePrefixPathAndTsFileVersionPair.right[1];
+ public TimeSeriesMetadataCacheKey(TsFileID tsFileID, String device, String measurement) {
+ this.regionId = tsFileID.regionId;
+ this.timePartitionId = tsFileID.timePartitionId;
+ this.tsFileVersion = tsFileID.fileVersion;
+ this.compactionVersion = tsFileID.compactionVersion;
this.device = device;
this.measurement = measurement;
}
+ public TimeSeriesMetadataCacheKey(
+ int regionId,
+ long timePartitionId,
+ long tsFileVersion,
+ long compactionVersion,
+ String device,
+ String measurement) {
+ this.regionId = regionId;
+ this.timePartitionId = timePartitionId;
+ this.tsFileVersion = tsFileVersion;
+ this.compactionVersion = compactionVersion;
+ this.device = device;
+ this.measurement = measurement;
+ }
+
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE
+ + sizeOfCharArray(device.length())
+ + sizeOfCharArray(measurement.length());
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -282,16 +301,38 @@ public class TimeSeriesMetadataCache {
return false;
}
TimeSeriesMetadataCacheKey that = (TimeSeriesMetadataCacheKey) o;
- return Objects.equals(measurement, that.measurement)
- && Objects.equals(device, that.device)
+ return regionId == that.regionId
+ && timePartitionId == that.timePartitionId
&& tsFileVersion == that.tsFileVersion
&& compactionVersion == that.compactionVersion
- && tsFilePrefixPath.equals(that.tsFilePrefixPath);
+ && Objects.equals(device, that.device)
+ && Objects.equals(measurement, that.measurement);
}
@Override
public int hashCode() {
- return Objects.hash(tsFilePrefixPath, tsFileVersion, compactionVersion, device, measurement);
+ return Objects.hash(
+ regionId, timePartitionId, tsFileVersion, compactionVersion, device, measurement);
+ }
+
+ @Override
+ public String toString() {
+ return "TimeSeriesMetadataCacheKey{"
+ + "regionId="
+ + regionId
+ + ", timePartitionId="
+ + timePartitionId
+ + ", tsFileVersion="
+ + tsFileVersion
+ + ", compactionVersion="
+ + compactionVersion
+ + ", device='"
+ + device
+ + '\''
+ + ", measurement='"
+ + measurement
+ + '\''
+ + '}';
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index ddb815f4d6e..b2a1b3511c0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -787,7 +787,7 @@ public class DataRegion implements IDataRegionForQuery {
for (Map<String, List<ChunkMetadata>> metaMap : writer.getMetadatasForQuery().values()) {
for (List<ChunkMetadata> metadatas : metaMap.values()) {
for (ChunkMetadata chunkMetadata : metadatas) {
- chunkMetadataSize += chunkMetadata.calculateRamSize();
+ chunkMetadataSize += chunkMetadata.getRetainedSizeInBytes();
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index a904f9844b2..f93fde5ab55 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -358,9 +358,6 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
TsFileSequenceReader reader = readerMap.get(tsFileResource);
List<AlignedChunkMetadata> alignedChunkMetadataList =
reader.getAlignedChunkMetadata(currentDevice.left);
- if (!alignedChunkMetadataList.isEmpty()) {
- alignedChunkMetadataList.forEach(x -> x.setFilePath(tsFileResource.getTsFilePath()));
- }
applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList);
readerAndChunkMetadataList.add(new Pair<>(reader, alignedChunkMetadataList));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
index 5b0165b2f7c..51eba154041 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -47,12 +48,15 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
// it's only exact while using limit & offset push down
private final boolean queryAllSensors;
+ private final TsFileResource resource;
+
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
- public DiskAlignedChunkLoader(boolean debug, boolean queryAllSensors) {
+ public DiskAlignedChunkLoader(boolean debug, boolean queryAllSensors, TsFileResource resource) {
this.debug = debug;
this.queryAllSensors = queryAllSensors;
+ this.resource = resource;
}
@Override
@@ -71,15 +75,31 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
long t1 = System.nanoTime();
try {
AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData;
+ ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata();
Chunk timeChunk =
ChunkCache.getInstance()
- .get((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(), debug);
+ .get(
+ new ChunkCache.ChunkCacheKey(
+ resource.getTsFilePath(),
+ resource.getTsFileID(),
+ timeChunkMetadata.getOffsetOfChunkHeader()),
+ timeChunkMetadata.getDeleteIntervalList(),
+ timeChunkMetadata.getStatistics(),
+ debug);
List<Chunk> valueChunkList = new ArrayList<>();
for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) {
valueChunkList.add(
valueChunkMetadata == null
? null
- : ChunkCache.getInstance().get((ChunkMetadata) valueChunkMetadata, debug));
+ : ChunkCache.getInstance()
+ .get(
+ new ChunkCache.ChunkCacheKey(
+ resource.getTsFilePath(),
+ resource.getTsFileID(),
+ valueChunkMetadata.getOffsetOfChunkHeader()),
+ valueChunkMetadata.getDeleteIntervalList(),
+ valueChunkMetadata.getStatistics(),
+ debug));
}
long t2 = System.nanoTime();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
index a58543587d5..a08b47f4313 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
@@ -37,18 +38,29 @@ import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.INI
/** To read one chunk from disk, and only used in iotdb server module. */
public class DiskChunkLoader implements IChunkLoader {
+ private final TsFileResource resource;
+
private final boolean debug;
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
- public DiskChunkLoader(boolean debug) {
+ public DiskChunkLoader(boolean debug, TsFileResource resource) {
this.debug = debug;
+ this.resource = resource;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData, debug);
+ return ChunkCache.getInstance()
+ .get(
+ new ChunkCache.ChunkCacheKey(
+ resource.getTsFilePath(),
+ resource.getTsFileID(),
+ chunkMetaData.getOffsetOfChunkHeader()),
+ chunkMetaData.getDeleteIntervalList(),
+ chunkMetaData.getStatistics(),
+ debug);
}
@Override
@@ -61,7 +73,16 @@ public class DiskChunkLoader implements IChunkLoader {
throws IOException {
long t1 = System.nanoTime();
try {
- Chunk chunk = ChunkCache.getInstance().get((ChunkMetadata) chunkMetaData, debug);
+ Chunk chunk =
+ ChunkCache.getInstance()
+ .get(
+ new ChunkCache.ChunkCacheKey(
+ resource.getTsFilePath(),
+ resource.getTsFileID(),
+ chunkMetaData.getOffsetOfChunkHeader()),
+ chunkMetaData.getDeleteIntervalList(),
+ chunkMetaData.getStatistics(),
+ debug);
long t2 = System.nanoTime();
IChunkReader chunkReader = new ChunkReader(chunk, timeFilter);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
index 9dfbf5a897e..0520200a9e8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
@@ -118,10 +118,10 @@ public class DiskAlignedChunkMetadataLoader implements IChunkMetadataLoader {
alignedChunkMetadataList.forEach(
chunkMetadata -> {
if (chunkMetadata.needSetChunkLoader()) {
- chunkMetadata.setFilePath(resource.getTsFilePath());
+ chunkMetadata.setVersion(resource.getVersion());
chunkMetadata.setClosed(resource.isClosed());
chunkMetadata.setChunkLoader(
- new DiskAlignedChunkLoader(context.isDebug(), queryAllSensors));
+ new DiskAlignedChunkLoader(context.isDebug(), queryAllSensors, resource));
}
});
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskChunkMetadataLoader.java
index b33a7e6fcc6..9e14b23a129 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -98,9 +98,9 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
chunkMetadataList.forEach(
chunkMetadata -> {
if (chunkMetadata.needSetChunkLoader()) {
- chunkMetadata.setFilePath(resource.getTsFilePath());
+ chunkMetadata.setVersion(resource.getVersion());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new DiskChunkLoader(context.isDebug()));
+ chunkMetadata.setChunkLoader(new DiskChunkLoader(context.isDebug(), resource));
}
});
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
index 17bd551c7ab..105e62620d8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
@@ -75,10 +75,10 @@ public class MemAlignedChunkMetadataLoader implements IChunkMetadataLoader {
chunkMetadataList.forEach(
chunkMetadata -> {
if (chunkMetadata.needSetChunkLoader()) {
- chunkMetadata.setFilePath(resource.getTsFilePath());
+ chunkMetadata.setVersion(resource.getVersion());
chunkMetadata.setClosed(resource.isClosed());
chunkMetadata.setChunkLoader(
- new DiskAlignedChunkLoader(context.isDebug(), queryAllSensors));
+ new DiskAlignedChunkLoader(context.isDebug(), queryAllSensors, resource));
}
});
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemChunkMetadataLoader.java
index 57d3a46a051..ebe499f2fe7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -66,9 +66,9 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
chunkMetadataList.forEach(
chunkMetadata -> {
if (chunkMetadata.needSetChunkLoader()) {
- chunkMetadata.setFilePath(resource.getTsFilePath());
+ chunkMetadata.setVersion(resource.getVersion());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new DiskChunkLoader(context.isDebug()));
+ chunkMetadata.setChunkLoader(new DiskChunkLoader(context.isDebug(), resource));
}
});
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
new file mode 100644
index 00000000000..9a315732074
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.tsfile;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.tsfile.utils.FilePathUtils.splitTsFilePath;
+
+public class TsFileID {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TsFileID.class);
+
+ public final int regionId;
+ public final long timePartitionId;
+ public final long fileVersion;
+ // high 32 bit is compaction level, low 32 bit is merge count
+ public final long compactionVersion;
+
+ public TsFileID() {
+ this.regionId = -1;
+ this.timePartitionId = -1;
+ this.fileVersion = -1;
+ this.compactionVersion = -1;
+ }
+
+ public TsFileID(int regionId, long timePartitionId, long fileVersion, long compactionVersion) {
+ this.regionId = regionId;
+ this.timePartitionId = timePartitionId;
+ this.fileVersion = fileVersion;
+ this.compactionVersion = compactionVersion;
+ }
+
+ public TsFileID(String tsFileAbsolutePath) {
+ int tmpRegionId = -1;
+ long tmpTimePartitionId = -1;
+ String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
+ int pathLength = pathSegments.length;
+ if (pathLength >= 3) {
+ try {
+ tmpRegionId = Integer.parseInt(pathSegments[pathLength - 3]);
+ } catch (NumberFormatException e) {
+ // ignore, load will get in here
+ }
+ try {
+ tmpTimePartitionId = Long.parseLong(pathSegments[pathLength - 2]);
+ } catch (NumberFormatException e) {
+ // ignore, load will get in here
+ }
+ }
+
+ this.regionId = tmpRegionId;
+ this.timePartitionId = tmpTimePartitionId;
+ long[] arr = splitAndGetVersionArray(pathSegments[pathLength - 1]);
+ this.fileVersion = arr[0];
+ this.compactionVersion = arr[1];
+ }
+
+ /**
+ * @return a long array whose length is 2, the first long value is tsfile version, second long
+ * value is compaction version, high 32 bit is in-space compaction count, low 32 bit is
+ * cross-space compaction count
+ */
+ private static long[] splitAndGetVersionArray(String tsFileName) {
+ String[] names = tsFileName.split(FILE_NAME_SEPARATOR);
+ long[] versionArray = new long[2];
+ if (names.length != 4) {
+ // ignore, load will get in here
+ return versionArray;
+ }
+ versionArray[0] = Long.parseLong(names[1]);
+
+ int dotIndex = names[3].indexOf(".");
+ versionArray[1] =
+ (Long.parseLong(names[2]) << 32) | Long.parseLong(names[3].substring(0, dotIndex));
+ return versionArray;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
index 8b1f6db60b0..53a6ee357b6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -35,20 +35,20 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class TsFileManager {
- private String storageGroupName;
+ private final String storageGroupName;
private String dataRegionId;
- private String storageGroupDir;
+ private final String storageGroupDir;
/** Serialize queries, delete resource files, compaction cleanup files */
private final ReadWriteLock resourceListLock = new ReentrantReadWriteLock();
private String writeLockHolder;
// time partition -> double linked list of tsfiles
- private TreeMap<Long, TsFileResourceList> sequenceFiles = new TreeMap<>();
- private TreeMap<Long, TsFileResourceList> unsequenceFiles = new TreeMap<>();
+ private final TreeMap<Long, TsFileResourceList> sequenceFiles = new TreeMap<>();
+ private final TreeMap<Long, TsFileResourceList> unsequenceFiles = new TreeMap<>();
private boolean allowCompaction = true;
- private AtomicLong currentCompactionTaskSerialId = new AtomicLong(0);
+ private final AtomicLong currentCompactionTaskSerialId = new AtomicLong(0);
public TsFileManager(String storageGroupName, String dataRegionId, String storageGroupDir) {
this.storageGroupName = storageGroupName;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 99c6aa710f6..c1721c999aa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeInd
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -57,15 +56,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -111,8 +106,6 @@ public class TsFileResource {
private TsFileLock tsFileLock = new TsFileLock();
- private final Random random = new Random();
-
private boolean isSeq;
private FSFactory fsFactory = FSFactoryProducer.getFSFactory();
@@ -125,7 +118,7 @@ public class TsFileResource {
/** Minimum index of plans executed within this TsFile. */
public long minPlanIndex = Long.MAX_VALUE;
- private long version = 0;
+ private TsFileID tsFileID;
private long ramSize;
@@ -158,32 +151,15 @@ public class TsFileResource {
private boolean isInsertionCompactionTaskCandidate = true;
- public TsFileResource() {}
-
- public TsFileResource(TsFileResource other) throws IOException {
- this.file = other.file;
- this.processor = other.processor;
- this.timeIndex = other.timeIndex;
- this.modFile = other.modFile;
- this.setAtomicStatus(other.getStatus());
- this.pathToChunkMetadataListMap = other.pathToChunkMetadataListMap;
- this.pathToReadOnlyMemChunkMap = other.pathToReadOnlyMemChunkMap;
- this.pathToTimeSeriesMetadataMap = other.pathToTimeSeriesMetadataMap;
- this.tsFileLock = other.tsFileLock;
- this.fsFactory = other.fsFactory;
- this.maxPlanIndex = other.maxPlanIndex;
- this.minPlanIndex = other.minPlanIndex;
- this.version = FilePathUtils.splitAndGetTsFileVersion(this.file.getName());
- this.tsFileSize = other.tsFileSize;
- this.isSeq = other.isSeq;
- this.tierLevel = other.tierLevel;
- this.maxProgressIndex = other.maxProgressIndex;
+ @TestOnly
+ public TsFileResource() {
+ this.tsFileID = new TsFileID();
}
/** for sealed TsFile, call setClosed to close TsFileResource */
public TsFileResource(File file) {
this.file = file;
- this.version = FilePathUtils.splitAndGetTsFileVersion(this.file.getName());
+ this.tsFileID = new TsFileID(file.getAbsolutePath());
this.timeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
this.isSeq = FilePathUtils.isSequence(this.file.getAbsolutePath());
// This method is invoked when DataNode recovers, so the tierLevel should be calculated when
@@ -200,7 +176,7 @@ public class TsFileResource {
/** unsealed TsFile, for writter */
public TsFileResource(File file, TsFileProcessor processor) {
this.file = file;
- this.version = FilePathUtils.splitAndGetTsFileVersion(this.file.getName());
+ this.tsFileID = new TsFileID(file.getAbsolutePath());
this.timeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
this.processor = processor;
this.isSeq = processor.isSequence();
@@ -221,18 +197,11 @@ public class TsFileResource {
this.pathToChunkMetadataListMap = pathToChunkMetadataListMap;
generatePathToTimeSeriesMetadataMap();
this.originTsFileResource = originTsFileResource;
- this.version = originTsFileResource.version;
+ this.tsFileID = originTsFileResource.tsFileID;
this.isSeq = originTsFileResource.isSeq;
this.tierLevel = originTsFileResource.tierLevel;
}
- @TestOnly
- public TsFileResource(
- File file, Map<String, Integer> deviceToIndex, long[] startTimes, long[] endTimes) {
- this.file = file;
- this.timeIndex = new DeviceTimeIndex(deviceToIndex, startTimes, endTimes);
- }
-
public synchronized void serialize() throws IOException {
FileOutputStream fileOutputStream = new FileOutputStream(file + RESOURCE_SUFFIX + TEMP_SUFFIX);
BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream);
@@ -369,6 +338,7 @@ public class TsFileResource {
public void setFile(File file) {
this.file = file;
+ this.tsFileID = new TsFileID(file.getAbsolutePath());
}
public File getTsFile() {
@@ -824,10 +794,6 @@ public class TsFileResource {
return null;
}
- public void setTimeSeriesMetadata(PartialPath path, ITimeSeriesMetadata timeSeriesMetadata) {
- this.pathToTimeSeriesMetadataMap.put(path, timeSeriesMetadata);
- }
-
public DataRegion.SettleTsFileCallBack getSettleTsFileCallBack() {
return settleTsFileCallBack;
}
@@ -838,7 +804,7 @@ public class TsFileResource {
/** make sure Either the deviceToIndex is not empty Or the path contains a partition folder */
public long getTimePartition() {
- return timeIndex.getTimePartition(file.getAbsolutePath());
+ return tsFileID.timePartitionId;
}
/**
@@ -855,48 +821,6 @@ public class TsFileResource {
return timeIndex.isSpanMultiTimePartitions();
}
- /**
- * Create a hardlink for the TsFile and modification file (if exists) The hardlink will have a
- * suffix like ".{sysTime}_{randomLong}"
- *
- * @return a new TsFileResource with its file changed to the hardlink or null the hardlink cannot
- * be created.
- */
- public TsFileResource createHardlink() {
- if (file == null || !file.exists()) {
- return null;
- }
-
- TsFileResource newResource;
- try {
- newResource = new TsFileResource(this);
- } catch (IOException e) {
- LOGGER.error("Cannot create hardlink for {}", file, e);
- return null;
- }
-
- while (true) {
- String hardlinkSuffix =
- TsFileConstant.PATH_SEPARATOR + System.currentTimeMillis() + "_" + random.nextLong();
- File hardlink = new File(file.getAbsolutePath() + hardlinkSuffix);
-
- try {
- Files.createLink(Paths.get(hardlink.getAbsolutePath()), Paths.get(file.getAbsolutePath()));
- newResource.setFile(hardlink);
- if (modFile != null && modFile.exists()) {
- newResource.setModFile(modFile.createHardlink());
- }
- break;
- } catch (FileAlreadyExistsException e) {
- // retry a different name if the file is already created
- } catch (IOException e) {
- LOGGER.error("Cannot create hardlink for {}", file, e);
- return null;
- }
- }
- return newResource;
- }
-
public void setModFile(ModificationFile modFile) {
synchronized (this) {
this.modFile = modFile;
@@ -966,11 +890,17 @@ public class TsFileResource {
}
public void setVersion(long version) {
- this.version = version;
+ this.tsFileID =
+ new TsFileID(
+ tsFileID.regionId, tsFileID.timePartitionId, version, tsFileID.compactionVersion);
}
public long getVersion() {
- return version;
+ return tsFileID.fileVersion;
+ }
+
+ public TsFileID getTsFileID() {
+ return tsFileID;
}
public void setTimeIndex(ITimeIndex timeIndex) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
index 3f99a9e8678..60f1e414b52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
@@ -250,8 +250,7 @@ public class TsFileOverlapValidationAndRepairTool {
continue;
}
- TsFileResource resource = new TsFileResource();
- resource.setFile(tsfile);
+ TsFileResource resource = new TsFileResource(tsfile);
resource.deserialize();
resource.close();
resources.add(resource);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index fe25fd0688a..7a08d5b3e39 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -141,16 +141,16 @@ public class FileLoaderUtils {
timeSeriesMetadata =
TimeSeriesMetadataCache.getInstance()
.get(
+ resource.getTsFilePath(),
new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
- resource.getTsFilePath(),
+ resource.getTsFileID(),
seriesPath.getDevice(),
seriesPath.getMeasurement()),
allSensors,
resource.getTimeIndexType() != 1,
context.isDebug());
if (timeSeriesMetadata != null) {
- List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(), seriesPath);
+ List<Modification> pathModifications = context.getPathModifications(resource, seriesPath);
timeSeriesMetadata.setModified(!pathModifications.isEmpty());
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, context, filter, pathModifications));
@@ -279,7 +279,8 @@ public class FileLoaderUtils {
// we should not ignore the non-exist of device in TsFileMetadata
TimeseriesMetadata timeColumn =
cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
+ filePath,
+ new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
allSensors,
resource.getTimeIndexType() != 1,
isDebug);
@@ -299,7 +300,9 @@ public class FileLoaderUtils {
for (String valueMeasurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
+ filePath,
+ new TimeSeriesMetadataCacheKey(
+ resource.getTsFileID(), deviceId, valueMeasurement),
allSensors,
resource.getTimeIndexType() != 1,
isDebug);
@@ -334,8 +337,7 @@ public class FileLoaderUtils {
for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
if (valueTimeSeriesMetadataList.get(i) != null) {
List<Modification> pathModifications =
- context.getPathModifications(
- resource.getModFile(), alignedPath.getPathWithMeasurement(i));
+ context.getPathModifications(resource, alignedPath.getPathWithMeasurement(i));
valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
res.add(pathModifications);
modified = (modified || !pathModifications.isEmpty());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesTestUtil.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesTestUtil.java
index fdf7ed18ce1..b500fbacaf7 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesTestUtil.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesTestUtil.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.execution.operator;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
@@ -73,6 +74,7 @@ public class AlignedSeriesTestUtil {
unseqResources.clear();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
@@ -90,7 +92,6 @@ public class AlignedSeriesTestUtil {
tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL);
tsFileResource.setMinPlanIndex(i);
tsFileResource.setMaxPlanIndex(i);
- tsFileResource.setVersion(i);
seqResources.add(tsFileResource);
prepareFile(sgName, tsFileResource, i * ptNum, ptNum, 0, measurementSchemas);
}
@@ -101,7 +102,6 @@ public class AlignedSeriesTestUtil {
tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL);
tsFileResource.setMinPlanIndex(i + seqFileNum);
tsFileResource.setMaxPlanIndex(i + seqFileNum);
- tsFileResource.setVersion(i + seqFileNum);
unseqResources.add(tsFileResource);
prepareFile(
sgName,
@@ -117,7 +117,6 @@ public class AlignedSeriesTestUtil {
tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL);
tsFileResource.setMinPlanIndex(seqFileNum + unseqFileNum);
tsFileResource.setMaxPlanIndex(seqFileNum + unseqFileNum);
- tsFileResource.setVersion(seqFileNum + unseqFileNum);
unseqResources.add(tsFileResource);
prepareFile(sgName, tsFileResource, 0, ptNum * 2, 20000, measurementSchemas);
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SortOperatorTest.java
index 8f30c5b5dc9..b7785248513 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SortOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SortOperatorTest.java
@@ -81,9 +81,12 @@ public class SortOperatorTest {
private int dataNodeId;
+ private int maxTsBlockSizeInBytes;
+
@Before
public void setUp() throws MetadataException, IOException, WriteProcessException {
dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
+ maxTsBlockSizeInBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(200);
SeriesReaderTestUtil.setUp(
@@ -94,6 +97,7 @@ public class SortOperatorTest {
public void tearDown() throws IOException {
SeriesReaderTestUtil.tearDown(seqResources, unSeqResources);
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(dataNodeId);
+ TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(maxTsBlockSizeInBytes);
}
// ------------------------------------------------------------------------------------------------
@@ -228,26 +232,31 @@ public class SortOperatorTest {
// with data spilling
@Test
public void sortOperatorSpillingTest() throws Exception {
- IoTDBDescriptor.getInstance().getConfig().setSortBufferSize(5000);
- SortOperator root = (SortOperator) genSortOperator(Ordering.ASC, true);
- int lastValue = -1;
- int count = 0;
- while (root.isBlocked().isDone() && root.hasNext()) {
- TsBlock tsBlock = root.next();
- if (tsBlock == null) continue;
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- long time = tsBlock.getTimeByIndex(i);
- int v1 = tsBlock.getColumn(0).getInt(i);
- int v2 = tsBlock.getColumn(1).getInt(i);
- assertTrue(lastValue == -1 || lastValue < v1);
- assertEquals(getValue(time), v1);
- assertEquals(v1, v2);
- lastValue = v1;
- count++;
+ long sortBufferSize = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+ try {
+ IoTDBDescriptor.getInstance().getConfig().setSortBufferSize(5000);
+ SortOperator root = (SortOperator) genSortOperator(Ordering.ASC, true);
+ int lastValue = -1;
+ int count = 0;
+ while (root.isBlocked().isDone() && root.hasNext()) {
+ TsBlock tsBlock = root.next();
+ if (tsBlock == null) continue;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ long time = tsBlock.getTimeByIndex(i);
+ int v1 = tsBlock.getColumn(0).getInt(i);
+ int v2 = tsBlock.getColumn(1).getInt(i);
+ assertTrue(lastValue == -1 || lastValue < v1);
+ assertEquals(getValue(time), v1);
+ assertEquals(v1, v2);
+ lastValue = v1;
+ count++;
+ }
}
+ root.close();
+ assertEquals(500, count);
+ } finally {
+ IoTDBDescriptor.getInstance().getConfig().setSortBufferSize(sortBufferSize);
}
- root.close();
- assertEquals(count, 500);
}
// no data spilling
@@ -271,6 +280,6 @@ public class SortOperatorTest {
}
}
root.close();
- assertEquals(count, 500);
+ assertEquals(500, count);
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/TopKOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/TopKOperatorTest.java
index 2aff973b154..166e7f2ea40 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/TopKOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/TopKOperatorTest.java
@@ -104,11 +104,14 @@ public class TopKOperatorTest {
private int dataNodeId;
+ private int maxTsBlockLineNumber;
+
private final int limitValue = 100;
@Before
public void setUp() throws MetadataException, IOException, WriteProcessException {
dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
+ maxTsBlockLineNumber = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
TSFileDescriptor.getInstance().getConfig().setMaxTsBlockLineNumber(3);
SeriesReaderTestUtil.setUp(
@@ -119,6 +122,7 @@ public class TopKOperatorTest {
public void tearDown() throws IOException {
SeriesReaderTestUtil.tearDown(seqResources, unSeqResources);
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(dataNodeId);
+ TSFileDescriptor.getInstance().getConfig().setMaxTsBlockLineNumber(maxTsBlockLineNumber);
}
long getValue(long expectedTime) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCacheTest.java
index dac861292c1..bea5c578ffd 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCacheTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.buffer;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -62,7 +63,7 @@ public class BloomFilterCacheTest {
.concat(File.separator)
.concat("root.sg" + (i + 1))
.concat(File.separator)
- .concat("0")
+ .concat(Integer.toString(i))
.concat(File.separator)
.concat("0")
.concat(File.separator)
@@ -82,6 +83,7 @@ public class BloomFilterCacheTest {
for (String filePath : pathList) {
FileUtils.forceDelete(new File(filePath));
}
+ bloomFilterCache.clear();
} catch (Exception e) {
Assert.fail(e.getMessage());
}
@@ -91,8 +93,15 @@ public class BloomFilterCacheTest {
public void testGet() {
try {
for (String filePath : pathList) {
+ TsFileID tsFileID = new TsFileID(filePath);
BloomFilter bloomFilter =
- bloomFilterCache.get(new BloomFilterCache.BloomFilterCacheKey(filePath));
+ bloomFilterCache.get(
+ new BloomFilterCache.BloomFilterCacheKey(
+ filePath,
+ tsFileID.regionId,
+ tsFileID.timePartitionId,
+ tsFileID.fileVersion,
+ tsFileID.compactionVersion));
TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
BloomFilter bloomFilter1 = reader.readBloomFilter();
Assert.assertEquals(bloomFilter1, bloomFilter);
@@ -108,7 +117,14 @@ public class BloomFilterCacheTest {
public void testRemove() {
try {
String path = pathList.get(0);
- BloomFilterCache.BloomFilterCacheKey key = new BloomFilterCache.BloomFilterCacheKey(path);
+ TsFileID tsFileID = new TsFileID(path);
+ BloomFilterCache.BloomFilterCacheKey key =
+ new BloomFilterCache.BloomFilterCacheKey(
+ path,
+ tsFileID.regionId,
+ tsFileID.timePartitionId,
+ tsFileID.fileVersion,
+ tsFileID.compactionVersion);
BloomFilter bloomFilter = bloomFilterCache.get(key);
TsFileSequenceReader reader = FileReaderManager.getInstance().get(path, true);
BloomFilter bloomFilter1 = reader.readBloomFilter();
@@ -127,7 +143,14 @@ public class BloomFilterCacheTest {
public void testClear() {
try {
for (String path : pathList) {
- BloomFilterCache.BloomFilterCacheKey key = new BloomFilterCache.BloomFilterCacheKey(path);
+ TsFileID tsFileID = new TsFileID(path);
+ BloomFilterCache.BloomFilterCacheKey key =
+ new BloomFilterCache.BloomFilterCacheKey(
+ path,
+ tsFileID.regionId,
+ tsFileID.timePartitionId,
+ tsFileID.fileVersion,
+ tsFileID.compactionVersion);
BloomFilter bloomFilter = bloomFilterCache.get(key);
TsFileSequenceReader reader = FileReaderManager.getInstance().get(path, true);
BloomFilter bloomFilter1 = reader.readBloomFilter();
@@ -136,7 +159,14 @@ public class BloomFilterCacheTest {
}
bloomFilterCache.clear();
for (String path : pathList) {
- BloomFilterCache.BloomFilterCacheKey key = new BloomFilterCache.BloomFilterCacheKey(path);
+ TsFileID tsFileID = new TsFileID(path);
+ BloomFilterCache.BloomFilterCacheKey key =
+ new BloomFilterCache.BloomFilterCacheKey(
+ path,
+ tsFileID.regionId,
+ tsFileID.timePartitionId,
+ tsFileID.fileVersion,
+ tsFileID.compactionVersion);
BloomFilter bloomFilter = bloomFilterCache.getIfPresent(key);
Assert.assertNull(bloomFilter);
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
index 7c92cd9ce1d..ec635c65a98 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/ChunkCacheTest.java
@@ -52,8 +52,6 @@ import java.util.ArrayList;
import java.util.List;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class ChunkCacheTest {
File tempSGDir;
@@ -91,43 +89,46 @@ public class ChunkCacheTest {
unseqResources.clear();
chunkCache.clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
@Test
public void testChunkCache() throws IOException {
TsFileResource tsFileResource = seqResources.get(0);
- TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
- List<Path> paths = reader.getAllPaths();
-
- ChunkMetadata firstChunkMetadata = reader.getChunkMetadataList(paths.get(0)).get(0);
- firstChunkMetadata.setFilePath(tsFileResource.getTsFilePath());
-
- // add cache
- chunkCache.getAverageSize();
- Chunk chunk1 = chunkCache.get(firstChunkMetadata);
- chunkCache.getAverageSize();
-
- ChunkMetadata chunkMetadataKey =
- new ChunkMetadata("sensor0", TSDataType.DOUBLE, 25, new DoubleStatistics());
- chunkMetadataKey.setVersion(0);
- chunkMetadataKey.setFilePath(tsFileResource.getTsFilePath());
-
- Assert.assertEquals(chunkMetadataKey, firstChunkMetadata);
-
- // get cache
- Chunk chunk2 = chunkCache.get(chunkMetadataKey);
- Assert.assertEquals(chunk1.getHeader(), chunk2.getHeader());
- Assert.assertEquals(chunk1.getData(), chunk2.getData());
-
- try {
- chunkMetadataKey.setFilePath(null);
- chunkCache.get(chunkMetadataKey);
- fail();
- } catch (NullPointerException e) {
- assertTrue(true);
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath())) {
+ List<Path> paths = reader.getAllPaths();
+
+ ChunkMetadata firstChunkMetadata = reader.getChunkMetadataList(paths.get(0)).get(0);
+
+ // add cache
+ Chunk chunk1 =
+ chunkCache.get(
+ new ChunkCache.ChunkCacheKey(
+ tsFileResource.getTsFilePath(),
+ tsFileResource.getTsFileID(),
+ firstChunkMetadata.getOffsetOfChunkHeader()),
+ firstChunkMetadata.getDeleteIntervalList(),
+ firstChunkMetadata.getStatistics(),
+ false);
+
+ ChunkMetadata chunkMetadataKey =
+ new ChunkMetadata("sensor0", TSDataType.DOUBLE, 25, new DoubleStatistics());
+ chunkMetadataKey.setVersion(0);
+
+ // get cache
+ Chunk chunk2 =
+ chunkCache.get(
+ new ChunkCache.ChunkCacheKey(
+ tsFileResource.getTsFilePath(),
+ tsFileResource.getTsFileID(),
+ chunkMetadataKey.getOffsetOfChunkHeader()),
+ chunkMetadataKey.getDeleteIntervalList(),
+ chunkMetadataKey.getStatistics(),
+ false);
+ Assert.assertEquals(chunk1.getHeader(), chunk2.getHeader());
+ Assert.assertEquals(chunk1.getData(), chunk2.getData());
}
- reader.close();
}
void prepareSeries() throws MetadataException {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java
deleted file mode 100644
index 1e7681f726b..00000000000
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.buffer;
-
-import org.junit.Test;
-
-public class TimeSeriesMetadataCacheTest {
-
- @Test
- public void testTimeSeriesMetadataCache() {}
-}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java
index 474c134cca9..7a7cdda7065 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java
@@ -445,6 +445,9 @@ public class AbstractCompactionTest {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldPagePointMaxNumber);
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(oldMaxDegreeOfIndexNode);
+ ChunkCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
if (SEQ_DIRS.exists()) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java
index 198e865dd07..7f18090f7f7 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerTest.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.CrossCompactionPerformer;
@@ -84,6 +85,9 @@ public class CompactionSchedulerTest {
@Before
public void setUp() throws MetadataException, IOException {
CompactionClearUtils.clearAllCompactionFiles();
+ ChunkCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
File basicOutputDir = new File(TestConstant.BASE_OUTPUT_PATH);
@@ -115,9 +119,10 @@ public class CompactionSchedulerTest {
public void tearDown() throws IOException, StorageEngineException {
CompactionTaskManager.getInstance().stop();
new CompactionConfigRestorer().restoreCompactionConfig();
+ CompactionClearUtils.clearAllCompactionFiles();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
- CompactionClearUtils.clearAllCompactionFiles();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
CompactionClearUtils.deleteEmptyDir(new File("target"));
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java
index fabd8fadf46..5a7071c425c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionSchedulerWithFastPerformerTest.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.CrossCompactionPerformer;
@@ -85,6 +86,9 @@ public class CompactionSchedulerWithFastPerformerTest {
@Before
public void setUp() throws MetadataException, IOException {
CompactionClearUtils.clearAllCompactionFiles();
+ ChunkCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
File basicOutputDir = new File(TestConstant.BASE_OUTPUT_PATH);
@@ -118,6 +122,7 @@ public class CompactionSchedulerWithFastPerformerTest {
new CompactionConfigRestorer().restoreCompactionConfig();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
CompactionClearUtils.clearAllCompactionFiles();
EnvironmentUtils.cleanAllDir();
CompactionClearUtils.deleteEmptyDir(new File("target"));
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
index c5c2d2a565c..557155f047c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
@@ -61,10 +61,13 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
private final String oldThreadName = Thread.currentThread().getName();
+ private long targetChunkSize;
+
@Before
public void setUp()
throws IOException, WriteProcessException, MetadataException, InterruptedException {
super.setUp();
+ targetChunkSize = IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
}
@@ -73,6 +76,7 @@ public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest {
public void tearDown() throws IOException, StorageEngineException {
super.tearDown();
Thread.currentThread().setName(oldThreadName);
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(targetChunkSize);
new CompactionConfigRestorer().restoreCompactionConfig();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
index 7db59c4e690..5a9b1058839 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
@@ -59,16 +59,22 @@ public class InsertionCrossSpaceCompactionTest extends AbstractCompactionTest {
new FixedPriorityBlockingQueue<>(50, new DefaultCompactionTaskComparatorImpl());
private final CompactionWorker worker = new CompactionWorker(0, candidateCompactionTaskQueue);
+ private boolean enableInsertionCrossSpaceCompaction;
+
@Before
public void setUp()
throws IOException, WriteProcessException, MetadataException, InterruptedException {
super.setUp();
+ enableInsertionCrossSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableInsertionCrossSpaceCompaction();
IoTDBDescriptor.getInstance().getConfig().setEnableInsertionCrossSpaceCompaction(true);
}
@After
public void tearDown() throws IOException, StorageEngineException {
- IoTDBDescriptor.getInstance().getConfig().setEnableInsertionCrossSpaceCompaction(false);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setEnableInsertionCrossSpaceCompaction(enableInsertionCrossSpaceCompaction);
super.tearDown();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
index 28c465a5453..b3d3b5ecb21 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.inner;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer;
@@ -85,6 +86,7 @@ public abstract class InnerCompactionTest {
unseqResources.clear();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
index e8c01e2b0e1..b8f7f7151bf 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -55,6 +56,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -101,6 +103,7 @@ public class InnerSeqCompactionWithFastPerformerTest {
CompactionClearUtils.clearAllCompactionFiles();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanEnv();
EnvironmentUtils.cleanAllDir();
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(prevMaxDegreeOfIndexNode);
@@ -1110,10 +1113,7 @@ public class InnerSeqCompactionWithFastPerformerTest {
pagePointsNum = new ArrayList<>();
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
- Set<String> paths = new HashSet<>();
- for (int i = 0; i < fullPaths.length; i++) {
- paths.add(fullPaths[i]);
- }
+ Set<String> paths = new HashSet<>(Arrays.asList(fullPaths));
for (int i = 0; i < 5; i++) {
TsFileResource tsFileResource =
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
index 83b5933c60e..f9ce303ecb9 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -100,6 +101,7 @@ public class InnerSeqCompactionWithReadChunkPerformerTest {
CompactionClearUtils.clearAllCompactionFiles();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanEnv();
EnvironmentUtils.cleanAllDir();
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(prevMaxDegreeOfIndexNode);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithFastPerformerTest.java
index 6208bd716b0..af8aaf19952 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithFastPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithFastPerformerTest.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.inner;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
@@ -107,6 +108,7 @@ public class InnerUnseqCompactionWithFastPerformerTest {
CompactionClearUtils.clearAllCompactionFiles();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
Thread.currentThread().setName(oldThreadName);
CompactionClearUtils.deleteEmptyDir(new File("target"));
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithReadPointPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithReadPointPerformerTest.java
index ec06ad0a941..d3b71e48df6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithReadPointPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerUnseqCompactionWithReadPointPerformerTest.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.inner;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
@@ -109,6 +110,7 @@ public class InnerUnseqCompactionWithReadPointPerformerTest {
CompactionClearUtils.clearAllCompactionFiles();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
Thread.currentThread().setName(oldThreadName);
CompactionClearUtils.deleteEmptyDir(new File("target"));
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
index 3272b34b7cc..8d5e7ee3c20 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
@@ -69,9 +69,7 @@ public class CompactionFileGeneratorUtils {
public static List<TsFileResource> getInnerCompactionTargetTsFileResources(
List<TsFileResource> fileResources, boolean seq) throws IOException {
List<TsFileResource> resources = new ArrayList<>();
- resources.add(
- new TsFileResource(
- TsFileNameGenerator.getInnerCompactionTargetFileResource(fileResources, seq)));
+ resources.add(TsFileNameGenerator.getInnerCompactionTargetFileResource(fileResources, seq));
return resources;
}
@@ -81,36 +79,7 @@ public class CompactionFileGeneratorUtils {
}
public static TsFileResource generateTsFileResource(boolean sequence, int index) {
- if (sequence) {
- return new TsFileResource(
- new File(
- TestConstant.BASE_OUTPUT_PATH
- .concat("database")
- .concat(File.separator)
- .concat("regionId")
- .concat(File.separator)
- .concat(
- index
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + index
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- } else {
- return new TsFileResource(
- new File(
- TestConstant.BASE_OUTPUT_PATH.concat(
- (index + 10000)
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + (index + 10000)
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- }
+ return generateTsFileResource(sequence, index, "default");
}
public static TsFileResource generateTsFileResource(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
index cbe7f9e3d22..afa5e3d6fb6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
@@ -231,12 +231,9 @@ public class TsFileProcessorTest {
assertEquals(entry1.getValue().size(), entry2.getValue().size());
for (int i = 0; i < entry1.getValue().size(); i++) {
ChunkMetadata chunkMetaData = entry1.getValue().get(i);
- chunkMetaData.setFilePath(filePath);
-
+ chunkMetaData.setVersion(0);
ChunkMetadata chunkMetadataRestore = entry2.getValue().get(i);
- chunkMetadataRestore.setFilePath(filePath);
-
- assertEquals(chunkMetaData, chunkMetadataRestore);
+ chunkMetadataRestore.setVersion(0);
}
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java
index e19a4ac640c..0e88aca64cb 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java
@@ -25,6 +25,9 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
+import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
+import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -432,6 +435,9 @@ public class AlignedSeriesScanLimitOffsetPushDownTest {
}
seqResources.clear();
unSeqResources.clear();
+ ChunkCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java
index 40953d5c1a5..4e257af290a 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.read.reader.series;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
@@ -100,6 +101,7 @@ public class SeriesReaderTestUtil {
unseqResources.clear();
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
index 13f67db170f..c3d57ce2c79 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
@@ -91,6 +92,7 @@ public class ResourceManagerTest {
tsFileResourceManager.setTimeIndexMemoryThreshold(prevTimeIndexMemoryThreshold);
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
+ BloomFilterCache.getInstance().clear();
TsFileResourceManager.getInstance().clear();
EnvironmentUtils.cleanAllDir();
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/sort/SortUtilTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/sort/SortUtilTest.java
index 6c64ae2552b..1209361e72b 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/sort/SortUtilTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/sort/SortUtilTest.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,11 +52,19 @@ public class SortUtilTest {
private static final String filePrefix = folderPath + File.separator + "tmp";
+ private int maxTsBlockSizeInBytes;
+
@Before
public void setUp() throws MetadataException, IOException, WriteProcessException {
+ maxTsBlockSizeInBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();
TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(200);
}
+ @After
+ public void tearDown() {
+ TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(maxTsBlockSizeInBytes);
+ }
+
private void clear() {
File tmpDir = new File(folderPath);
if (!tmpDir.exists()) return;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 5f8d0db3d36..9b7c37782b3 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -269,12 +269,6 @@ public class IoTDBConstant {
// cross space compaction
public static final String CROSS_COMPACTION_TMP_FILE_SUFFIX = ".cross";
- // cross space compaction of previous version (<0.13)
- public static final String CROSS_COMPACTION_TMP_FILE_SUFFIX_FROM_OLD = ".merge";
-
- // compaction mods of previous version (<0.13)
- public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods";
-
public static final String SETTLE_SUFFIX = ".settle";
public static final String MODS_SETTLE_FILE_SUFFIX = ".mods.settle";
public static final String BLANK = "";
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index 9bb531fb5a4..5494de5e3e6 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -31,6 +31,8 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -38,6 +40,12 @@ import java.nio.ByteBuffer;
public class ChunkHeader {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(ChunkHeader.class).instanceSize()
+ + ClassLayout.parseClass(TSDataType.class).instanceSize()
+ + ClassLayout.parseClass(CompressionType.class).instanceSize()
+ + ClassLayout.parseClass(TSEncoding.class).instanceSize();
+
/**
* 1 means this chunk has more than one page, so each page has its own page statistic. 5 means
* this chunk has only one page, and this page has no page statistic.
@@ -49,13 +57,13 @@ public class ChunkHeader {
private String measurementID;
private int dataSize;
- private TSDataType dataType;
- private CompressionType compressionType;
- private TSEncoding encodingType;
+ private final TSDataType dataType;
+ private final CompressionType compressionType;
+ private final TSEncoding encodingType;
// the following fields do not need to be serialized.
private int numOfPages;
- private int serializedSize;
+ private final int serializedSize;
public ChunkHeader(
String measurementID,
@@ -130,7 +138,7 @@ public class ChunkHeader {
+ ReadWriteForEncodingUtils.varIntSize(measurementIdLength) // measurementID length
+ measurementIdLength // measurementID
+ ReadWriteForEncodingUtils.uVarIntSize(dataSize) // dataSize
- + TSDataType.getSerializedSize() // dataType
+ + TSDataType.getSerializedSize() // dataTypex
+ CompressionType.getSerializedSize() // compressionType
+ TSEncoding.getSerializedSize(); // encodingType
}
@@ -178,27 +186,39 @@ public class ChunkHeader {
*
* @param input TsFileInput
* @param offset offset
- * @param chunkHeaderSize the estimated size of chunk's header
* @return CHUNK_HEADER object
* @throws IOException IOException
*/
- public static ChunkHeader deserializeFrom(TsFileInput input, long offset, int chunkHeaderSize)
- throws IOException {
+ public static ChunkHeader deserializeFrom(TsFileInput input, long offset) throws IOException {
- // read chunk header from input to buffer
- ByteBuffer buffer = ByteBuffer.allocate(chunkHeaderSize);
+ ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES + Integer.BYTES + 1);
input.read(buffer, offset);
buffer.flip();
+ // read chunk header from input to buffer
byte chunkType = buffer.get();
+ int strLength = ReadWriteForEncodingUtils.readVarInt(buffer);
+ int alreadyReadLength = buffer.position();
+
+ int remainingBytes =
+ strLength
+ + Integer.BYTES
+ + 1 // uVarInt dataSize
+ + TSDataType.getSerializedSize() // dataType
+ + CompressionType.getSerializedSize() // compressionType
+ + TSEncoding.getSerializedSize();
+ buffer = ByteBuffer.allocate(remainingBytes);
+
+ input.read(buffer, offset + alreadyReadLength);
+ buffer.flip();
+
// read measurementID
- String measurementID = ReadWriteIOUtils.readVarIntString(buffer);
+ String measurementID = ReadWriteIOUtils.readStringWithLength(buffer, strLength);
int dataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
TSDataType dataType = ReadWriteIOUtils.readDataType(buffer);
CompressionType type = ReadWriteIOUtils.readCompressionType(buffer);
TSEncoding encoding = ReadWriteIOUtils.readEncoding(buffer);
- chunkHeaderSize =
- chunkHeaderSize - Integer.BYTES - 1 + ReadWriteForEncodingUtils.uVarIntSize(dataSize);
+ int chunkHeaderSize = alreadyReadLength + buffer.position();
return new ChunkHeader(
chunkType, measurementID, dataSize, chunkHeaderSize, dataType, type, encoding);
}
@@ -225,6 +245,10 @@ public class ChunkHeader {
return measurementID;
}
+ public void setMeasurementID(String measurementID) {
+ this.measurementID = measurementID;
+ }
+
public int getDataSize() {
return dataSize;
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
index 18d8ac6fcc7..e840b8bf3e7 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
@@ -144,16 +144,6 @@ public class AlignedChunkMetadata implements IChunkMetadata {
this.chunkLoader = chunkLoader;
}
- @Override
- public void setFilePath(String filePath) {
- timeChunkMetadata.setFilePath(filePath);
- for (IChunkMetadata chunkMetadata : valueChunkMetadataList) {
- if (chunkMetadata != null) {
- chunkMetadata.setFilePath(filePath);
- }
- }
- }
-
@Override
public void setClosed(boolean closed) {
timeChunkMetadata.setClosed(closed);
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 26af7f5e3c9..68e93cc5825 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -23,11 +23,11 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.utils.FilePathUtils;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
@@ -39,6 +39,11 @@ import java.util.Objects;
/** Metadata of one chunk. */
public class ChunkMetadata implements IChunkMetadata {
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(ChunkMetadata.class).instanceSize()
+ + ClassLayout.parseClass(TSDataType.class).instanceSize()
+ + ClassLayout.parseClass(ArrayList.class).instanceSize();
+
private String measurementUid;
/**
@@ -55,7 +60,7 @@ public class ChunkMetadata implements IChunkMetadata {
private long version;
/** A list of deleted intervals. */
- private List<TimeRange> deleteIntervalList;
+ private ArrayList<TimeRange> deleteIntervalList;
private boolean modified;
@@ -64,23 +69,15 @@ public class ChunkMetadata implements IChunkMetadata {
private Statistics<? extends Serializable> statistics;
- private long ramSize;
-
private static final int CHUNK_METADATA_FIXED_RAM_SIZE = 93;
// used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata
private boolean isSeq = true;
private boolean isClosed;
- private String filePath;
// 0x80 for time chunk, 0x40 for value chunk, 0x00 for common chunk
private byte mask;
- // used for ChunkCache, Eg:"root.sg1/0/0"
- private String tsFilePrefixPath;
- // high 32 bit is compaction level, low 32 bit is merge count
- private long compactionVersion;
-
public ChunkMetadata() {}
/**
@@ -108,23 +105,17 @@ public class ChunkMetadata implements IChunkMetadata {
this.offsetOfChunkHeader = other.offsetOfChunkHeader;
this.tsDataType = other.tsDataType;
this.version = other.version;
- this.chunkLoader = other.chunkLoader;
this.statistics = other.statistics;
- this.ramSize = other.ramSize;
this.isSeq = other.isSeq;
this.isClosed = other.isClosed;
- this.filePath = other.filePath;
this.mask = other.mask;
- this.tsFilePrefixPath = other.tsFilePrefixPath;
- this.compactionVersion = other.compactionVersion;
}
@Override
public String toString() {
return String.format(
- "measurementId: %s, datatype: %s, version: %d, "
- + "Statistics: %s, deleteIntervalList: %s, filePath: %s",
- measurementUid, tsDataType, version, statistics, deleteIntervalList, filePath);
+ "measurementId: %s, datatype: %s, version: %d, " + "Statistics: %s, deleteIntervalList: %s",
+ measurementUid, tsDataType, version, statistics, deleteIntervalList);
}
public long getNumOfPoints() {
@@ -227,7 +218,7 @@ public class ChunkMetadata implements IChunkMetadata {
@Override
public void insertIntoSortedDeletions(TimeRange timeRange) {
- List<TimeRange> resultInterval = new ArrayList<>();
+ ArrayList<TimeRange> resultInterval = new ArrayList<>();
long startTime = timeRange.getMin();
long endTime = timeRange.getMax();
if (deleteIntervalList != null) {
@@ -269,26 +260,6 @@ public class ChunkMetadata implements IChunkMetadata {
this.chunkLoader = chunkLoader;
}
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ChunkMetadata that = (ChunkMetadata) o;
- return offsetOfChunkHeader == that.offsetOfChunkHeader
- && version == that.version
- && compactionVersion == that.compactionVersion
- && tsFilePrefixPath.equals(that.tsFilePrefixPath);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(tsFilePrefixPath, version, compactionVersion, offsetOfChunkHeader);
- }
-
@Override
public boolean isModified() {
return modified;
@@ -299,14 +270,6 @@ public class ChunkMetadata implements IChunkMetadata {
this.modified = modified;
}
- public long calculateRamSize() {
- long memSize = CHUNK_METADATA_FIXED_RAM_SIZE;
- memSize += RamUsageEstimator.sizeOf(tsFilePrefixPath);
- memSize += RamUsageEstimator.sizeOf(measurementUid);
- memSize += statistics.calculateRamSize();
- return memSize;
- }
-
public static long calculateRamSize(String measurementId, TSDataType dataType) {
return CHUNK_METADATA_FIXED_RAM_SIZE
+ RamUsageEstimator.sizeOf(measurementId)
@@ -316,7 +279,11 @@ public class ChunkMetadata implements IChunkMetadata {
public void mergeChunkMetadata(ChunkMetadata chunkMetadata) {
Statistics<? extends Serializable> chunkMetadataStatistics = chunkMetadata.getStatistics();
this.statistics.mergeStatistics(chunkMetadataStatistics);
- this.ramSize = calculateRamSize();
+ }
+
+ // it's only used for query cache, measurementUid is from TimeSeriesMetadata
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE + (statistics == null ? 0L : statistics.getRetainedSizeInBytes());
}
@Override
@@ -337,21 +304,6 @@ public class ChunkMetadata implements IChunkMetadata {
isClosed = closed;
}
- public String getFilePath() {
- return filePath;
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
-
- Pair<String, long[]> tsFilePrefixPathAndTsFileVersionPair =
- FilePathUtils.getTsFilePrefixPathAndTsFileVersionPair(filePath);
- // set tsFilePrefixPath
- tsFilePrefixPath = tsFilePrefixPathAndTsFileVersionPair.left;
- this.version = tsFilePrefixPathAndTsFileVersionPair.right[0];
- this.compactionVersion = tsFilePrefixPathAndTsFileVersionPair.right[1];
- }
-
@Override
public byte getMask() {
return mask;
@@ -360,4 +312,36 @@ public class ChunkMetadata implements IChunkMetadata {
public void setMask(byte mask) {
this.mask = mask;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ChunkMetadata that = (ChunkMetadata) o;
+ return offsetOfChunkHeader == that.offsetOfChunkHeader
+ && version == that.version
+ && isSeq == that.isSeq
+ && isClosed == that.isClosed
+ && mask == that.mask
+ && Objects.equals(measurementUid, that.measurementUid)
+ && tsDataType == that.tsDataType
+ && Objects.equals(statistics, that.statistics);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ measurementUid,
+ offsetOfChunkHeader,
+ tsDataType,
+ version,
+ statistics,
+ isSeq,
+ isClosed,
+ mask);
+ }
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
index 8700a7b796a..acd3b46986c 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
@@ -57,8 +57,6 @@ public interface IChunkMetadata {
void setChunkLoader(IChunkLoader chunkLoader);
- void setFilePath(String filePath);
-
void setClosed(boolean closed);
TSDataType getDataType();
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2c878613c67..3eff99b4a26 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -26,6 +26,8 @@ import org.apache.iotdb.tsfile.utils.PublicBAOS;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
@@ -33,12 +35,17 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
+
+import static io.airlift.slice.SizeOf.sizeOfCharArray;
+import static io.airlift.slice.SizeOf.sizeOfObjectArray;
public class TimeseriesMetadata implements ITimeSeriesMetadata {
- /** used for old version tsfile. */
- private long startOffsetOfChunkMetaDataList;
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(TimeseriesMetadata.class).instanceSize()
+ + ClassLayout.parseClass(String.class).instanceSize()
+ + ClassLayout.parseClass(TSDataType.class).instanceSize()
+ + ClassLayout.parseClass(ArrayList.class).instanceSize();
/**
* 0 means this time series has only one chunk, no need to save the statistic again in chunk
@@ -97,7 +104,9 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
this.dataType = timeseriesMetadata.dataType;
this.statistics = timeseriesMetadata.statistics;
this.modified = timeseriesMetadata.modified;
- this.chunkMetadataList = new ArrayList<>(timeseriesMetadata.chunkMetadataList);
+ // we won't change the list in query process so it's safe to directly assign it without copying
+ // new one
+ this.chunkMetadataList = timeseriesMetadata.chunkMetadataList;
}
public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata) {
@@ -186,14 +195,6 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
this.timeSeriesMetadataType = timeSeriesMetadataType;
}
- public long getOffsetOfChunkMetaDataList() {
- return startOffsetOfChunkMetaDataList;
- }
-
- public void setOffsetOfChunkMetaDataList(long position) {
- this.startOffsetOfChunkMetaDataList = position;
- }
-
public String getMeasurementId() {
return measurementId;
}
@@ -241,9 +242,11 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
}
public List<IChunkMetadata> getCopiedChunkMetadataList() {
- return chunkMetadataList.stream()
- .map(chunkMetadata -> new ChunkMetadata((ChunkMetadata) chunkMetadata))
- .collect(Collectors.toList());
+ List<IChunkMetadata> res = new ArrayList<>(chunkMetadataList.size());
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ res.add(new ChunkMetadata((ChunkMetadata) chunkMetadata));
+ }
+ return res;
}
@Override
@@ -276,12 +279,28 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
this.chunkMetadataList = new ArrayList<>(chunkMetadataList);
}
+ // it's only used for query cache, field chunkMetadataListBuffer and chunkMetadataLoader should
+ // always be null
+ public long getRetainedSizeInBytes() {
+ long retainedSize =
+ INSTANCE_SIZE
+ + sizeOfCharArray(measurementId.length())
+ + (statistics == null ? 0 : statistics.getRetainedSizeInBytes());
+ int length = chunkMetadataList == null ? 0 : chunkMetadataList.size();
+ if (length > 0) {
+ retainedSize += sizeOfObjectArray(length);
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ retainedSize +=
+ chunkMetadata == null ? 0 : ((ChunkMetadata) chunkMetadata).getRetainedSizeInBytes();
+ }
+ }
+ return retainedSize;
+ }
+
@Override
public String toString() {
return "TimeseriesMetadata{"
- + "startOffsetOfChunkMetaDataList="
- + startOffsetOfChunkMetaDataList
- + ", timeSeriesMetadataType="
+ + "timeSeriesMetadataType="
+ timeSeriesMetadataType
+ ", chunkMetaDataListDataSize="
+ chunkMetaDataListDataSize
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index e34caf38b96..f7b44310fcd 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -28,7 +26,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
-import java.util.Set;
/** TSFileMetaData collects all metadata info and saves in its data structure. */
public class TsFileMetadata {
@@ -100,19 +97,6 @@ public class TsFileMetadata {
return byteLen;
}
- /**
- * use the given outputStream to serialize bloom filter.
- *
- * @param outputStream -output stream to determine byte length
- * @return -byte length
- * @throws IOException error when operating outputStream
- */
- public int buildAndSerializeBloomFilter(OutputStream outputStream, Set<Path> paths)
- throws IOException {
- BloomFilter filter = buildBloomFilter(paths);
- return serializeBloomFilter(outputStream, filter);
- }
-
public int serializeBloomFilter(OutputStream outputStream, BloomFilter filter)
throws IOException {
int byteLen = 0;
@@ -126,21 +110,6 @@ public class TsFileMetadata {
return byteLen;
}
- /**
- * build bloom filter.
- *
- * @return bloom filter
- */
- private BloomFilter buildBloomFilter(Set<Path> paths) {
- BloomFilter filter =
- BloomFilter.getEmptyBloomFilter(
- TSFileDescriptor.getInstance().getConfig().getBloomFilterErrorRate(), paths.size());
- for (Path path : paths) {
- filter.add(path.toString());
- }
- return filter;
- }
-
public long getMetaOffset() {
return metaOffset;
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
index 5cf423ad2c9..99e960568ec 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -23,21 +23,27 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Objects;
+import static io.airlift.slice.SizeOf.sizeOfCharArray;
+
/** Statistics for string type. */
public class BinaryStatistics extends Statistics<Binary> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(BinaryStatistics.class).instanceSize()
+ + 2 * ClassLayout.parseClass(Binary.class).instanceSize();
+
private Binary firstValue = new Binary("", TSFileConfig.STRING_CHARSET);
private Binary lastValue = new Binary("", TSFileConfig.STRING_CHARSET);
- static final int BINARY_STATISTICS_FIXED_RAM_SIZE = 32;
@Override
public TSDataType getType() {
@@ -50,6 +56,13 @@ public class BinaryStatistics extends Statistics<Binary> {
return 4 + firstValue.getValues().length + 4 + lastValue.getValues().length;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE
+ + sizeOfCharArray(firstValue.getLength())
+ + sizeOfCharArray(lastValue.getLength());
+ }
+
/**
* initialize Statistics.
*
@@ -143,11 +156,6 @@ public class BinaryStatistics extends Statistics<Binary> {
}
}
- @Override
- public long calculateRamSize() {
- return RamUsageEstimator.sizeOf(this);
- }
-
@Override
public int serializeStats(OutputStream outputStream) throws IOException {
int byteLen = 0;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
index 5d5f60b9d3a..c5aa84ada3c 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,12 +33,13 @@ import java.util.Objects;
public class BooleanStatistics extends Statistics<Boolean> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(BooleanStatistics.class).instanceSize();
+
private boolean firstValue;
private boolean lastValue;
private long sumValue;
- static final int BOOLEAN_STATISTICS_FIXED_RAM_SIZE = 56;
-
@Override
public TSDataType getType() {
return TSDataType.BOOLEAN;
@@ -48,6 +51,11 @@ public class BooleanStatistics extends Statistics<Boolean> {
return 10;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
/**
* initialize boolean Statistics.
*
@@ -96,11 +104,6 @@ public class BooleanStatistics extends Statistics<Boolean> {
}
}
- @Override
- public long calculateRamSize() {
- return BOOLEAN_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
public Boolean getMinValue() {
throw new StatisticsClassException(
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
index ae09809cbd1..79e9775fa39 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,14 +33,15 @@ import java.util.Objects;
public class DoubleStatistics extends Statistics<Double> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(DoubleStatistics.class).instanceSize();
+
private double minValue;
private double maxValue;
private double firstValue;
private double lastValue;
private double sumValue;
- static final int DOUBLE_STATISTICS_FIXED_RAM_SIZE = 80;
-
@Override
public TSDataType getType() {
return TSDataType.DOUBLE;
@@ -50,6 +53,11 @@ public class DoubleStatistics extends Statistics<Double> {
return 40;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
/**
* initialize double statistics.
*
@@ -121,11 +129,6 @@ public class DoubleStatistics extends Statistics<Double> {
}
}
- @Override
- public long calculateRamSize() {
- return DOUBLE_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
public Double getMinValue() {
return minValue;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
index d976426e96e..9b9d4032cd3 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -22,6 +22,8 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,14 +33,15 @@ import java.util.Objects;
/** Statistics for float type. */
public class FloatStatistics extends Statistics<Float> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(FloatStatistics.class).instanceSize();
+
private float minValue;
private float maxValue;
private float firstValue;
private float lastValue;
private double sumValue;
- static final int FLOAT_STATISTICS_FIXED_RAM_SIZE = 64;
-
@Override
public TSDataType getType() {
return TSDataType.FLOAT;
@@ -50,6 +53,11 @@ public class FloatStatistics extends Statistics<Float> {
return 24;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
public void initializeStats(float min, float max, float first, float last, double sum) {
this.minValue = min;
this.maxValue = max;
@@ -112,11 +120,6 @@ public class FloatStatistics extends Statistics<Float> {
}
}
- @Override
- public long calculateRamSize() {
- return FLOAT_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
public Float getMinValue() {
return minValue;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
index 31bde55dd67..715f509b4f8 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -32,14 +34,15 @@ import java.util.Objects;
/** Statistics for int type. */
public class IntegerStatistics extends Statistics<Integer> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(IntegerStatistics.class).instanceSize();
+
private int minValue;
private int maxValue;
private int firstValue;
private int lastValue;
private long sumValue;
- static final int INTEGER_STATISTICS_FIXED_RAM_SIZE = 64;
-
@Override
public TSDataType getType() {
return TSDataType.INT32;
@@ -51,6 +54,11 @@ public class IntegerStatistics extends Statistics<Integer> {
return 24;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
public void initializeStats(int min, int max, int first, int last, long sum) {
this.minValue = min;
this.maxValue = max;
@@ -113,11 +121,6 @@ public class IntegerStatistics extends Statistics<Integer> {
}
}
- @Override
- public long calculateRamSize() {
- return INTEGER_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
public Integer getMinValue() {
return minValue;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
index 629363e997e..031a7f3b7b3 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,14 +33,15 @@ import java.util.Objects;
public class LongStatistics extends Statistics<Long> {
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(LongStatistics.class).instanceSize();
+
private long minValue;
private long maxValue;
private long firstValue;
private long lastValue;
private double sumValue;
- static final int LONG_STATISTICS_FIXED_RAM_SIZE = 80;
-
@Override
public TSDataType getType() {
return TSDataType.INT64;
@@ -50,6 +53,11 @@ public class LongStatistics extends Statistics<Long> {
return 40;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
public void initializeStats(long min, long max, long firstValue, long last, double sum) {
this.minValue = min;
this.maxValue = max;
@@ -153,11 +161,6 @@ public class LongStatistics extends Statistics<Long> {
}
}
- @Override
- public long calculateRamSize() {
- return LONG_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
protected void mergeStatisticsValue(Statistics<Long> stats) {
LongStatistics longStats = (LongStatistics) stats;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index fa9ad67ea66..96bb55a09e6 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -93,19 +93,19 @@ public abstract class Statistics<T extends Serializable> {
public static int getSizeByType(TSDataType type) {
switch (type) {
case INT32:
- return IntegerStatistics.INTEGER_STATISTICS_FIXED_RAM_SIZE;
+ return IntegerStatistics.INSTANCE_SIZE;
case INT64:
- return LongStatistics.LONG_STATISTICS_FIXED_RAM_SIZE;
+ return LongStatistics.INSTANCE_SIZE;
case TEXT:
- return BinaryStatistics.BINARY_STATISTICS_FIXED_RAM_SIZE;
+ return BinaryStatistics.INSTANCE_SIZE;
case BOOLEAN:
- return BooleanStatistics.BOOLEAN_STATISTICS_FIXED_RAM_SIZE;
+ return BooleanStatistics.INSTANCE_SIZE;
case DOUBLE:
- return DoubleStatistics.DOUBLE_STATISTICS_FIXED_RAM_SIZE;
+ return DoubleStatistics.INSTANCE_SIZE;
case FLOAT:
- return FloatStatistics.FLOAT_STATISTICS_FIXED_RAM_SIZE;
+ return FloatStatistics.INSTANCE_SIZE;
case VECTOR:
- return TimeStatistics.TIME_STATISTICS_FIXED_RAM_SIZE;
+ return TimeStatistics.INSTANCE_SIZE;
default:
throw new UnknownColumnTypeException(type.toString());
}
@@ -121,6 +121,8 @@ public abstract class Statistics<T extends Serializable> {
public abstract int getStatsSize();
+ public abstract long getRetainedSizeInBytes();
+
public int serialize(OutputStream outputStream) throws IOException {
int byteLen = 0;
byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
@@ -390,8 +392,6 @@ public abstract class Statistics<T extends Serializable> {
this.count = count;
}
- public abstract long calculateRamSize();
-
public boolean containedByTimeFilter(Filter timeFilter) {
return timeFilter == null || timeFilter.containStartEndTime(getStartTime(), getEndTime());
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java
index 1531652b423..f1d9d78c7d1 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java
@@ -22,6 +22,8 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -29,7 +31,9 @@ import java.nio.ByteBuffer;
public class TimeStatistics extends Statistics<Long> {
- static final int TIME_STATISTICS_FIXED_RAM_SIZE = 40;
+ public static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(TimeStatistics.class).instanceSize();
+
private static final String TIME = "Time";
private static final String UPDATE_STATS = "update stats";
@@ -44,6 +48,11 @@ public class TimeStatistics extends Statistics<Long> {
return 0;
}
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
@Override
public void update(long time) {
super.update(time);
@@ -111,11 +120,6 @@ public class TimeStatistics extends Statistics<Long> {
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, UPDATE_STATS));
}
- @Override
- public long calculateRamSize() {
- return TIME_STATISTICS_FIXED_RAM_SIZE;
- }
-
@Override
protected void mergeStatisticsValue(Statistics<Long> stats) {}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 6d8c534af48..83bdac644d6 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1265,11 +1265,10 @@ public class TsFileSequenceReader implements AutoCloseable {
* read the chunk's header.
*
* @param position the file offset of this chunk's header
- * @param chunkHeaderSize the size of chunk's header
*/
- private ChunkHeader readChunkHeader(long position, int chunkHeaderSize) throws IOException {
+ private ChunkHeader readChunkHeader(long position) throws IOException {
try {
- return ChunkHeader.deserializeFrom(tsFileInput, position, chunkHeaderSize);
+ return ChunkHeader.deserializeFrom(tsFileInput, position);
} catch (Throwable t) {
logger.warn("Exception {} happened while reading chunk header of {}", t.getMessage(), file);
throw t;
@@ -1292,6 +1291,22 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
+ /**
+ * read memory chunk.
+ *
+ * @return -chunk
+ */
+ public Chunk readMemChunk(long offset) throws IOException {
+ try {
+ ChunkHeader header = readChunkHeader(offset);
+ ByteBuffer buffer = readChunk(offset + header.getSerializedSize(), header.getDataSize());
+ return new Chunk(header, buffer);
+ } catch (Throwable t) {
+ logger.warn("Exception {} happened while reading chunk of {}", t.getMessage(), file);
+ throw t;
+ }
+ }
+
/**
* read memory chunk.
*
@@ -1300,8 +1315,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public Chunk readMemChunk(ChunkMetadata metaData) throws IOException {
try {
- int chunkHeadSize = ChunkHeader.getSerializedSize(metaData.getMeasurementUid());
- ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), chunkHeadSize);
+ ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader());
ByteBuffer buffer =
readChunk(
metaData.getOffsetOfChunkHeader() + header.getSerializedSize(), header.getDataSize());
@@ -1319,8 +1333,7 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return chunk
*/
public Chunk readMemChunk(CachedChunkLoaderImpl.ChunkCacheKey chunkCacheKey) throws IOException {
- int chunkHeadSize = ChunkHeader.getSerializedSize(chunkCacheKey.getMeasurementUid());
- ChunkHeader header = readChunkHeader(chunkCacheKey.getOffsetOfChunkHeader(), chunkHeadSize);
+ ChunkHeader header = readChunkHeader(chunkCacheKey.getOffsetOfChunkHeader());
ByteBuffer buffer =
readChunk(
chunkCacheKey.getOffsetOfChunkHeader() + header.getSerializedSize(),
@@ -1357,8 +1370,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return null;
}
IChunkMetadata lastChunkMetadata = chunkMetadataList.get(chunkMetadataList.size() - 1);
- int chunkHeadSize = ChunkHeader.getSerializedSize(lastChunkMetadata.getMeasurementUid());
- ChunkHeader header = readChunkHeader(lastChunkMetadata.getOffsetOfChunkHeader(), chunkHeadSize);
+ ChunkHeader header = readChunkHeader(lastChunkMetadata.getOffsetOfChunkHeader());
return new MeasurementSchema(
lastChunkMetadata.getMeasurementUid(),
header.getDataType(),
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
index 8a8d21dcdae..e0b07a33239 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
@@ -25,16 +25,26 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.openjdk.jol.info.ClassLayout;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
+import static io.airlift.slice.SizeOf.sizeOfByteArray;
+
/** used in query. */
public class Chunk {
- private ChunkHeader chunkHeader;
- private Statistics chunkStatistic;
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(Chunk.class).instanceSize()
+ + ChunkHeader.INSTANCE_SIZE
+ + ClassLayout.parseClass(ByteBuffer.class).instanceSize();
+
+ private final ChunkHeader chunkHeader;
private ByteBuffer chunkData;
+ private Statistics chunkStatistic;
+
/** A list of deleted intervals. */
private List<TimeRange> deleteIntervalList;
@@ -49,6 +59,11 @@ public class Chunk {
this.chunkStatistic = chunkStatistic;
}
+ public Chunk(ChunkHeader header, ByteBuffer buffer) {
+ this.chunkHeader = header;
+ this.chunkData = buffer;
+ }
+
public ChunkHeader getHeader() {
return chunkHeader;
}
@@ -148,4 +163,13 @@ public class Chunk {
public Statistics getChunkStatistic() {
return chunkStatistic;
}
+
+ /**
+ * it's only used for query cache, and assuming that we use HeapByteBuffer, if we use Pooled
+ * DirectByteBuffer in the future, we need to change the calculation logic here. chunkStatistic
+ * and deleteIntervalList are all null in cache
+ */
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE + sizeOfByteArray(chunkData.capacity());
+ }
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java
index 36a4fcdf098..4b21f8487bd 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java
@@ -20,40 +20,37 @@ package org.apache.iotdb.tsfile.utils;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import java.util.Arrays;
+import org.openjdk.jol.info.ClassLayout;
+
import java.util.BitSet;
import java.util.Objects;
+import static io.airlift.slice.SizeOf.sizeOfLongArray;
+
public class BloomFilter {
+ private static final int INSTANCE_SIZE =
+ ClassLayout.parseClass(BloomFilter.class).instanceSize()
+ + ClassLayout.parseClass(BitSet.class).instanceSize();
+
private static final int MINIMAL_SIZE = 256;
private static final int MAXIMAL_HASH_FUNCTION_SIZE = 8;
private static final int[] SEEDS = new int[] {5, 7, 11, 19, 31, 37, 43, 59};
- private int size;
- private int hashFunctionSize;
- private BitSet bits;
- private HashFunction[] func;
+
+ private final int size;
+ private final int hashFunctionSize;
+ private final BitSet bits;
// do not try to initialize the filter by construction method
private BloomFilter(byte[] bytes, int size, int hashFunctionSize) {
this.size = size;
this.hashFunctionSize = hashFunctionSize;
- func = new HashFunction[hashFunctionSize];
- for (int i = 0; i < hashFunctionSize; i++) {
- func[i] = new HashFunction(size, SEEDS[i]);
- }
-
bits = BitSet.valueOf(bytes);
}
private BloomFilter(int size, int hashFunctionSize) {
this.size = size;
this.hashFunctionSize = hashFunctionSize;
- func = new HashFunction[hashFunctionSize];
- for (int i = 0; i < hashFunctionSize; i++) {
- func[i] = new HashFunction(size, SEEDS[i]);
- }
-
bits = new BitSet(size);
}
@@ -93,13 +90,9 @@ public class BloomFilter {
return size;
}
- public void setSize(int size) {
- this.size = size;
- }
-
public void add(String value) {
- for (HashFunction f : func) {
- bits.set(f.hash(value), true);
+ for (int i = 0; i < hashFunctionSize; i++) {
+ bits.set(hash(value, size, SEEDS[i]), true);
}
}
@@ -110,25 +103,22 @@ public class BloomFilter {
boolean ret = true;
int index = 0;
while (ret && index < hashFunctionSize) {
- ret = bits.get(func[index++].hash(value));
+ ret = bits.get(hash(value, size, SEEDS[index++]));
}
return ret;
}
- public int getBitCount() {
- int res = 0;
- for (int i = 0; i < size; i++) {
- res += bits.get(i) ? 1 : 0;
- }
-
- return res;
- }
-
public byte[] serialize() {
return bits.toByteArray();
}
+ public long getRetainedSizeInBytes() {
+ // calculated according to the implementation of BitSet, bits.size() returns words.length *
+ // BITS_PER_WORD and BITS_PER_WORD = (1 << 6) which is 64
+ return INSTANCE_SIZE + sizeOfLongArray(bits.size() / 64);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -140,49 +130,20 @@ public class BloomFilter {
BloomFilter that = (BloomFilter) o;
return size == that.size
&& hashFunctionSize == that.hashFunctionSize
- && Objects.equals(bits, that.bits)
- && Arrays.equals(func, that.func);
+ && Objects.equals(bits, that.bits);
}
@Override
public int hashCode() {
- return Objects.hash(size, hashFunctionSize, bits, func);
+ return Objects.hash(size, hashFunctionSize, bits);
}
- private class HashFunction {
-
- private int cap;
- private int seed;
-
- HashFunction(int cap, int seed) {
- this.cap = cap;
- this.seed = seed;
- }
-
- public int hash(String value) {
- int res = Murmur128Hash.hash(value, seed);
- if (res == Integer.MIN_VALUE) {
- res = 0;
- }
-
- return Math.abs(res) % cap;
+ private static int hash(String value, int cap, int seed) {
+ int res = Murmur128Hash.hash(value, seed);
+ if (res == Integer.MIN_VALUE) {
+ res = 0;
}
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- HashFunction that = (HashFunction) o;
- return cap == that.cap && seed == that.seed;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(cap, seed);
- }
+ return Math.abs(res) % cap;
}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
index df7ac44ef8e..3a4904f9ed6 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
@@ -23,8 +23,6 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import java.io.File;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
-
public class FilePathUtils {
private static final String LOCAL_PATH_SPLIT_STRING = "\\".equals(File.separator) ? "\\\\" : "/";
@@ -120,24 +118,6 @@ public class FilePathUtils {
return Long.parseLong(names[1]);
}
- /**
- * @return a long array whose length is 2, the first long value is tsfile version, second long
- * value is compaction version, high 32 bit is in-space compaction count, low 32 bit is
- * cross-space compaction count
- */
- private static long[] splitAndGetVersionArray(String tsFileName) {
- String[] names = tsFileName.split(FILE_NAME_SEPARATOR);
- long[] versionArray = new long[2];
- if (names.length != 4) {
- return versionArray;
- }
- versionArray[0] = Long.parseLong(names[1]);
- versionArray[1] =
- (Long.parseLong(names[2]) << 32)
- | Long.parseLong(names[3].substring(0, names[3].length() - TSFILE_SUFFIX.length()));
- return versionArray;
- }
-
public static Pair<String, Long> getLogicalSgNameAndTimePartitionIdPair(
String tsFileAbsolutePath) {
String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
@@ -145,22 +125,4 @@ public class FilePathUtils {
pathSegments[pathSegments.length - 4],
Long.parseLong(pathSegments[pathSegments.length - 2]));
}
-
- /**
- * pair.left tsFilePrefixPath, like data/data/sequence/root.sg1/0/0 pair.right is a long array
- * whose length is 2 pair.right[0] is tsfile version pair.right[1] is compaction version, high 32
- * bit is compaction level, low 32 bit is merge count
- */
- public static Pair<String, long[]> getTsFilePrefixPathAndTsFileVersionPair(
- String tsFileAbsolutePath) {
- String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
- int pathLength = pathSegments.length;
- return new Pair<>(
- pathSegments[pathLength - 4]
- + File.separator
- + pathSegments[pathLength - 3]
- + File.separator
- + pathSegments[pathLength - 2],
- splitAndGetVersionArray(pathSegments[pathLength - 1]));
- }
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
index 38eb3417533..a0c33254817 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
@@ -659,6 +659,11 @@ public class ReadWriteIOUtils {
/** read string from byteBuffer with user define length. */
public static String readStringWithLength(ByteBuffer buffer, int length) {
+ if (length < 0) {
+ return null;
+ } else if (length == 0) {
+ return "";
+ }
byte[] bytes = new byte[length];
buffer.get(bytes, 0, length);
return new String(bytes, 0, length);
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3677dc828fd..b56fa6fe3bb 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -75,7 +75,7 @@ public class TsFileIOWriter implements AutoCloseable {
protected static final byte[] MAGIC_STRING_BYTES;
public static final byte VERSION_NUMBER_BYTE;
- protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+ protected static final TSFileConfig TS_FILE_CONFIG = TSFileDescriptor.getInstance().getConfig();
private static final Logger logger = LoggerFactory.getLogger(TsFileIOWriter.class);
private static final Logger resourceLogger = LoggerFactory.getLogger("FileMonitor");
@@ -307,7 +307,7 @@ public class TsFileIOWriter implements AutoCloseable {
/** end chunk and write some log. */
public void endCurrentChunk() {
if (enableMemoryControl) {
- this.currentChunkMetadataSize += currentChunkMetadata.calculateRamSize();
+ this.currentChunkMetadataSize += currentChunkMetadata.getRetainedSizeInBytes();
}
chunkMetadataCount++;
chunkMetadataList.add(currentChunkMetadata);
@@ -379,13 +379,11 @@ public class TsFileIOWriter implements AutoCloseable {
Path currentPath = null;
MetadataIndexNode currentIndexNode =
new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
- TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
int seriesIdxForCurrDevice = 0;
BloomFilter filter =
BloomFilter.getEmptyBloomFilter(
TSFileDescriptor.getInstance().getConfig().getBloomFilterErrorRate(), pathCount);
- int indexCount = 0;
while (tsmIterator.hasNext()) {
// read in all chunk metadata of one series
// construct the timeseries metadata for this series
@@ -393,7 +391,6 @@ public class TsFileIOWriter implements AutoCloseable {
TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right;
currentPath = timeseriesMetadataPair.left;
- indexCount++;
// build bloom filter
filter.add(currentPath.getFullPath());
// construct the index tree node for the series
@@ -412,7 +409,7 @@ public class TsFileIOWriter implements AutoCloseable {
seriesIdxForCurrDevice = 0;
}
- if (seriesIdxForCurrDevice % config.getMaxDegreeOfIndexNode() == 0) {
+ if (seriesIdxForCurrDevice % TS_FILE_CONFIG.getMaxDegreeOfIndexNode() == 0) {
if (currentIndexNode.isFull()) {
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
index a72c564f982..064ffb83b1d 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java
@@ -60,7 +60,6 @@ public class TestHelper {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setMeasurementId(measurementuid);
timeseriesMetaData.setTsDataType(PageHeaderTest.DATA_TYPE);
- timeseriesMetaData.setOffsetOfChunkMetaDataList(1000L);
timeseriesMetaData.setDataSizeOfChunkMetaDataList(0);
timeseriesMetaData.setChunkMetadataListBuffer(new PublicBAOS());
timeseriesMetaData.setStatistics(statistics);