You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/01/20 10:23:20 UTC
[iotdb] 02/02: wait for upgrade tool
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFileV2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c76d13519e284f02d56d4aacc5a7056396aa678e
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Jan 20 18:22:31 2021 +0800
wait for upgrade tool
---
.../resources/conf/iotdb-engine.properties | 6 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +-
.../db/engine/cache/CacheHitRatioMonitor.java | 25 ---
.../engine/cache/CacheHitRatioMonitorMXBean.java | 11 -
.../apache/iotdb/db/engine/cache/ChunkCache.java | 12 +-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 222 ---------------------
.../db/engine/cache/TimeSeriesMetadataCache.java | 12 +-
.../db/engine/compaction/TsFileManagement.java | 2 -
.../level/LevelCompactionTsFileManagement.java | 2 -
.../iotdb/db/engine/merge/task/MergeFileTask.java | 4 -
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 -
.../db/query/reader/chunk/DiskChunkLoader.java | 11 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 33 ++-
.../chunk/metadata/MemChunkMetadataLoader.java | 6 -
.../org/apache/iotdb/db/service/TSServiceImpl.java | 6 +-
.../db/engine/cache/ChunkMetadataCacheTest.java | 172 ----------------
.../db/engine/compaction/LevelCompactionTest.java | 3 -
.../apache/iotdb/db/engine/merge/MergeTest.java | 2 -
.../iotdb/db/integration/IoTDBClearCacheIT.java | 3 -
.../query/reader/series/SeriesReaderTestUtil.java | 2 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 -
.../apache/iotdb/spark/db/EnvironmentUtils.java | 21 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 19 ++
.../tsfile/file/metadata/TimeseriesMetadata.java | 27 ++-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 143 +++++--------
.../read/controller/IChunkMetadataLoader.java | 8 -
.../file/metadata/TimeSeriesMetadataTest.java | 2 +-
28 files changed, 134 insertions(+), 648 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 8201e9d..73fa1da 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -380,9 +380,9 @@ query_time_threshold=60000
# whether to cache meta data(ChunkMetadata and TimeSeriesMetadata) or not.
meta_data_cache_enable=true
-# Read memory Allocation Ratio: ChunkMetadataCache, ChunkCache, TimeSeriesMetadataCache, memory used for constructing QueryDataSet and Free Memory Used in Query.
-# The parameter form is a:b:c:d:e, where a, b, c, d and e are integers. for example: 1:1:1:1:1 , 1:1:1:3:4
-chunkmeta_chunk_timeseriesmeta_free_memory_proportion=1:1:1:3:4
+# Read memory Allocation Ratio: ChunkCache, TimeSeriesMetadataCache, memory used for constructing QueryDataSet and Free Memory Used in Query.
+# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 1:2:3:4
+chunk_timeseriesmeta_free_memory_proportion=1:2:3:4
# cache size for MManager.
# This cache is used to improve insert speed where all path check and TSDataType will be cached in MManager with corresponding Path.
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 0871744..7463691 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -409,12 +409,8 @@ public class IoTDBConfig {
/**
* Memory allocated for timeSeriesMetaData cache in read process
*/
- private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 10;
+ private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 5;
- /**
- * Memory allocated for chunkMetaData cache in read process
- */
- private long allocateMemoryForChunkMetaDataCache = allocateMemoryForRead / 10;
/**
* Memory allocated for chunk cache in read process
@@ -1683,14 +1679,6 @@ public class IoTDBConfig {
this.allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForTimeSeriesMetaDataCache;
}
- public long getAllocateMemoryForChunkMetaDataCache() {
- return allocateMemoryForChunkMetaDataCache;
- }
-
- public void setAllocateMemoryForChunkMetaDataCache(long allocateMemoryForChunkMetaDataCache) {
- this.allocateMemoryForChunkMetaDataCache = allocateMemoryForChunkMetaDataCache;
- }
-
public long getAllocateMemoryForChunkCache() {
return allocateMemoryForChunkCache;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b11f9d3..0c5c19e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -835,7 +835,7 @@ public class IoTDBDescriptor {
}
String queryMemoryAllocateProportion = properties
- .getProperty("chunkmeta_chunk_timeseriesmeta_free_memory_proportion");
+ .getProperty("chunk_timeseriesmeta_free_memory_proportion");
if (queryMemoryAllocateProportion != null) {
String[] proportions = queryMemoryAllocateProportion.split(":");
int proportionSum = 0;
@@ -845,14 +845,12 @@ public class IoTDBDescriptor {
long maxMemoryAvailable = conf.getAllocateMemoryForRead();
if (proportionSum != 0) {
try {
- conf.setAllocateMemoryForChunkMetaDataCache(
- maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
conf.setAllocateMemoryForChunkCache(
- maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
+ maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
conf.setAllocateMemoryForTimeSeriesMetaDataCache(
- maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum);
+ maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
conf.setAllocateMemoryForReadWithoutCache(
- maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum);
+ maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum);
} catch (Exception e) {
throw new RuntimeException(
"Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion"
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
index 2c411ac..df078ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
@@ -53,31 +53,6 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public double getChunkMetaDataHitRatio() {
- return ChunkMetadataCache.getInstance().calculateChunkMetaDataHitRatio();
- }
-
- @Override
- public long getChunkMetaDataCacheUsedMemory() {
- return ChunkMetadataCache.getInstance().getUsedMemory();
- }
-
- @Override
- public long getChunkMetaDataCacheMaxMemory() {
- return ChunkMetadataCache.getInstance().getMaxMemory();
- }
-
- @Override
- public double getChunkMetaDataCacheUsedMemoryProportion() {
- return ChunkMetadataCache.getInstance().getUsedMemoryProportion();
- }
-
- @Override
- public long getChunkMetaDataCacheAverageSize() {
- return ChunkMetadataCache.getInstance().getAverageSize();
- }
-
- @Override
public double getChunkHitRatio() {
return ChunkCache.getInstance().calculateChunkHitRatio();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
index 0514481..ae2697b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
@@ -20,17 +20,6 @@ package org.apache.iotdb.db.engine.cache;
public interface CacheHitRatioMonitorMXBean {
- double getChunkMetaDataHitRatio();
-
- long getChunkMetaDataCacheUsedMemory();
-
- long getChunkMetaDataCacheMaxMemory();
-
- double getChunkMetaDataCacheUsedMemoryProportion();
-
- long getChunkMetaDataCacheAverageSize();
-
-
double getChunkHitRatio();
long getChunkCacheUsedMemory();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index c621a26..345d2e0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -25,6 +25,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -83,8 +84,11 @@ public class ChunkCache {
return ChunkCacheHolder.INSTANCE;
}
- public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader) throws IOException {
+ public Chunk get(ChunkMetadata chunkMetaData) throws IOException {
if (!CACHE_ENABLE) {
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed());
Chunk chunk = reader.readMemChunk(chunkMetaData);
return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeleteIntervalList(), chunkMetaData.getStatistics());
@@ -104,6 +108,8 @@ public class ChunkCache {
printCacheLog(true);
} else {
printCacheLog(false);
+ TsFileSequenceReader reader = FileReaderManager.getInstance()
+ .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed());
try {
chunk = reader.readMemChunk(chunkMetaData);
} catch (IOException e) {
@@ -112,7 +118,9 @@ public class ChunkCache {
}
lock.writeLock().lock();
try {
- lruCache.put(chunkMetaData, chunk);
+ if (!lruCache.containsKey(chunkMetaData)) {
+ lruCache.put(chunkMetaData, chunk);
+ }
} finally {
lock.writeLock().unlock();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
deleted file mode 100644
index f8fd5f3..0000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.TestOnly;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching
- * strategy is LRU.
- */
-public class ChunkMetadataCache {
-
- private static final Logger logger = LoggerFactory.getLogger(ChunkMetadataCache.class);
- private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
- private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChunkMetaDataCache();
- private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
-
- /**
- * key: file path dot deviceId dot sensorId.
- * <p>
- * value: chunkMetaData list of one timeseries in the file.
- */
- private final LRULinkedHashMap<AccountableString, List<ChunkMetadata>> lruCache;
-
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private final AtomicLong cacheHitNum = new AtomicLong();
- private final AtomicLong cacheRequestNum = new AtomicLong();
-
-
- private ChunkMetadataCache(long memoryThreshold) {
- if (CACHE_ENABLE) {
- logger.info("ChunkMetadataCache size = " + memoryThreshold);
- }
- lruCache = new LRULinkedHashMap<AccountableString, List<ChunkMetadata>>(memoryThreshold) {
- @Override
- protected long calEntrySize(AccountableString key, List<ChunkMetadata> value) {
- if (value.isEmpty()) {
- return RamUsageEstimator.sizeOf(key) + RamUsageEstimator.shallowSizeOf(value);
- }
- long entrySize;
- if (count < 10) {
- long currentSize = value.get(0).calculateRamSize();
- averageSize = ((averageSize * count) + currentSize) / (++count);
- entrySize = RamUsageEstimator.sizeOf(key)
- + (currentSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size()
- + RamUsageEstimator.shallowSizeOf(value);
- } else if (count < 100000) {
- count++;
- entrySize = RamUsageEstimator.sizeOf(key)
- + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size()
- + RamUsageEstimator.shallowSizeOf(value);
- } else {
- averageSize = value.get(0).calculateRamSize();
- count = 1;
- entrySize = RamUsageEstimator.sizeOf(key)
- + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size()
- + RamUsageEstimator.shallowSizeOf(value);
- }
- return entrySize;
- }
- };
- }
-
- public static ChunkMetadataCache getInstance() {
- return ChunkMetadataCacheSingleton.INSTANCE;
- }
-
- /**
- * get {@link ChunkMetadata}. THREAD SAFE.
- */
- public List<ChunkMetadata> get(String filePath, Path seriesPath,
- TimeseriesMetadata timeseriesMetadata) throws IOException {
- if (timeseriesMetadata == null) {
- return Collections.emptyList();
- }
- if (!CACHE_ENABLE) {
- // bloom filter part
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
- // If timeseries isn't included in the tsfile, empty list is returned.
- return tsFileReader.readChunkMetaDataList(timeseriesMetadata);
- }
-
- AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR
- + seriesPath.getDevice() + IoTDBConstant.PATH_SEPARATOR + seriesPath.getMeasurement());
-
- cacheRequestNum.incrementAndGet();
-
- lock.readLock().lock();
- List<ChunkMetadata> chunkMetadataList;
- try {
- chunkMetadataList = lruCache.get(key);
- } finally {
- lock.readLock().unlock();
- }
-
- if (chunkMetadataList != null) {
- printCacheLog(true);
- cacheHitNum.incrementAndGet();
- } else {
- printCacheLog(false);
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
- chunkMetadataList = tsFileReader.readChunkMetaDataList(timeseriesMetadata);
- lock.writeLock().lock();
- try {
- lruCache.put(key, chunkMetadataList);
- } finally {
- lock.writeLock().unlock();
- }
- }
- if (config.isDebugOn()) {
- DEBUG_LOGGER.info(
- "Chunk meta data list size: " + chunkMetadataList.size() + " key is: " + key.getString());
- chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
- }
- return new ArrayList<>(chunkMetadataList);
- }
-
- private void printCacheLog(boolean isHit) {
- if (!logger.isDebugEnabled()) {
- return;
- }
- logger.debug(
- "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.",
- isHit ? "" : "didn't ", cacheRequestNum.get(),
- cacheHitNum.get() * 1.0 / cacheRequestNum.get());
- }
-
- double calculateChunkMetaDataHitRatio() {
- if (cacheRequestNum.get() != 0) {
- return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
- } else {
- return 0;
- }
- }
-
- public long getUsedMemory() {
- return lruCache.getUsedMemory();
- }
-
- public long getMaxMemory() {
- return lruCache.getMaxMemory();
- }
-
- public double getUsedMemoryProportion() {
- return lruCache.getUsedMemoryProportion();
- }
-
- public long getAverageSize() {
- return lruCache.getAverageSize();
- }
-
- /**
- * clear LRUCache.
- */
- public void clear() {
- lock.writeLock().lock();
- if (lruCache != null) {
- lruCache.clear();
- }
- lock.writeLock().unlock();
- }
-
- public void remove(TsFileResource resource) {
- lock.writeLock().lock();
- if (resource != null) {
- lruCache.entrySet()
- .removeIf(e -> e.getKey().getString().startsWith(resource.getTsFilePath()));
- }
- lock.writeLock().unlock();
- }
-
- @TestOnly
- public boolean isEmpty() {
- return lruCache.isEmpty();
- }
-
- /**
- * singleton pattern.
- */
- private static class ChunkMetadataCacheSingleton {
-
- private static final ChunkMetadataCache INSTANCE = new
- ChunkMetadataCache(MEMORY_THRESHOLD_IN_B);
- }
-}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index a0cf793..b329aa5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -83,7 +83,10 @@ public class TimeSeriesMetadataCache {
currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(key.device)
+ RamUsageEstimator.sizeOf(key.measurement) + RamUsageEstimator.shallowSizeOf(value)
+ RamUsageEstimator.sizeOf(value.getMeasurementId()) + RamUsageEstimator
- .shallowSizeOf(value.getStatistics());
+ .shallowSizeOf(value.getStatistics()) +
+ (value.getChunkMetadataList().get(0).calculateRamSize()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.getChunkMetadataList().size()
+ + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList());
averageSize = ((averageSize * count) + currentSize) / (++count);
} else if (count < 100000) {
count++;
@@ -92,7 +95,10 @@ public class TimeSeriesMetadataCache {
averageSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(key.device)
+ RamUsageEstimator.sizeOf(key.measurement) + RamUsageEstimator.shallowSizeOf(value)
+ RamUsageEstimator.sizeOf(value.getMeasurementId()) + RamUsageEstimator
- .shallowSizeOf(value.getStatistics());
+ .shallowSizeOf(value.getStatistics()) +
+ (value.getChunkMetadataList().get(0).calculateRamSize()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.getChunkMetadataList().size()
+ + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList());
count = 1;
currentSize = averageSize;
}
@@ -193,7 +199,7 @@ public class TimeSeriesMetadataCache {
"Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device,
key.measurement, key.filePath, timeseriesMetadata);
}
- return timeseriesMetadata;
+ return new TimeseriesMetadata(timeseriesMetadata);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index 3d8e755..3d88e70 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -32,7 +32,6 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
@@ -322,7 +321,6 @@ public abstract class TsFileManagement {
// clean cache
if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
}
} finally {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index ec1ebdd..35c7f7d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -41,7 +41,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.compaction.TsFileManagement;
import org.apache.iotdb.db.engine.compaction.utils.CompactionLogAnalyzer;
import org.apache.iotdb.db.engine.compaction.utils.CompactionLogger;
@@ -123,7 +122,6 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
private void deleteLevelFile(TsFileResource seqFile) {
seqFile.writeLock();
try {
- ChunkMetadataCache.getInstance().remove(seqFile);
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
seqFile.setDeleted(true);
seqFile.delete();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index a21e1a0..fbf6bc1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeContext;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
@@ -143,7 +142,6 @@ class MergeFileTask {
seqFile.writeLock();
try {
- ChunkMetadataCache.getInstance().remove(seqFile);
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
resource.removeFileReader(seqFile);
@@ -294,7 +292,6 @@ class MergeFileTask {
seqFile.writeLock();
try {
resource.removeFileReader(seqFile);
- ChunkMetadataCache.getInstance().remove(seqFile);
File newMergeFile = seqFile.getTsFile();
newMergeFile.delete();
@@ -306,7 +303,6 @@ class MergeFileTask {
// clean cache
if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index b3416a3..0ee2ce8 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -71,7 +71,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.flush.pool.FlushTaskPoolManager;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
@@ -337,7 +336,6 @@ public class PlanExecutor implements IPlanExecutor {
private void operateClearCache() {
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 60ceab0..4e946ce 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -21,10 +21,7 @@ package org.apache.iotdb.db.query.reader.chunk;
import java.io.IOException;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
@@ -33,17 +30,13 @@ import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
*/
public class DiskChunkLoader implements IChunkLoader {
- private final TsFileResource resource;
+ public DiskChunkLoader() {
- public DiskChunkLoader(TsFileResource resource) {
- this.resource = resource;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- TsFileSequenceReader tsFileSequenceReader =
- FileReaderManager.getInstance().get(resource.getTsFilePath(), resource.isClosed());
- return ChunkCache.getInstance().get(chunkMetaData, tsFileSequenceReader);
+ return ChunkCache.getInstance().get(chunkMetaData);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 2b55d3b..09a7e0e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.query.reader.chunk.metadata;
-import java.io.IOException;
import java.util.List;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -33,11 +32,11 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
- private TsFileResource resource;
- private PartialPath seriesPath;
- private QueryContext context;
+ private final TsFileResource resource;
+ private final PartialPath seriesPath;
+ private final QueryContext context;
// time filter or value filter, only used to check time range
- private Filter filter;
+ private final Filter filter;
public DiskChunkMetadataLoader(TsFileResource resource, PartialPath seriesPath,
QueryContext context, Filter filter) {
@@ -69,18 +68,6 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
return chunkMetadataList;
}
- /**
- * For query v0.9/v1 tsfile only When generate temporary timeseriesMetadata set DiskChunkLoader to
- * each chunkMetadata in the List
- *
- * @param chunkMetadataList
- * @throws IOException
- */
- @Override
- public void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) {
- setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
- }
-
public static void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList,
TsFileResource resource, PartialPath seriesPath, QueryContext context) {
List<Modification> pathModifications =
@@ -89,10 +76,14 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
-
- for (ChunkMetadata data : chunkMetadataList) {
- data.setChunkLoader(new DiskChunkLoader(resource));
- }
+ // it is ok, even if it is not thread safe, because the cost of creating a DiskChunkLoader is very cheap.
+ chunkMetadataList.forEach(chunkMetadata -> {
+ if (chunkMetadata.getChunkLoader() == null) {
+ chunkMetadata.setFilePath(resource.getTsFilePath());
+ chunkMetadata.setClosed(resource.isClosed());
+ chunkMetadata.setChunkLoader(new DiskChunkLoader());
+ }
+ });
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 396cb24..843a982 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.query.reader.chunk.metadata;
-import java.io.IOException;
import java.util.List;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -69,9 +68,4 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
}
return chunkMetadataList;
}
-
- @Override
- public void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) throws IOException {
- // DO NOTHING
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 261d0f5..d7a4739 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -49,7 +49,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.cost.statistic.Measurement;
import org.apache.iotdb.db.cost.statistic.Operation;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.IoTDBException;
@@ -664,11 +663,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
if (config.isDebugOn()) {
SLOW_SQL_LOGGER.info(
- "ChunkCache used memory proportion: {}\nChunkMetadataCache used memory proportion: {}\n"
+ "ChunkCache used memory proportion: {}\n"
+ "TimeSeriesMetadataCache used memory proportion: {}",
ChunkCache.getInstance()
- .getUsedMemoryProportion(),
- ChunkMetadataCache.getInstance().getUsedMemoryProportion(), TimeSeriesMetadataCache
+ .getUsedMemoryProportion(), TimeSeriesMetadataCache
.getInstance().getUsedMemoryProportion());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
deleted file mode 100644
index 7241405..0000000
--- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.constant.TestConstant;
-import org.apache.iotdb.db.engine.MetadataManagerHelper;
-import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
-import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.WriteProcessException;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
-import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.metadata.PartialPath;
-import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.EnvironmentUtils;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ChunkMetadataCacheTest {
-
- private QueryContext context = EnvironmentUtils.TEST_QUERY_CONTEXT;
-
- private String storageGroup = "root.vehicle.d0";
- private String measurementId0 = "s0";
- private String measurementId1 = "s1";
- private String measurementId2 = "s2";
- private String measurementId3 = "s3";
- private String measurementId4 = "s4";
- private String measurementId5 = "s5";
- private StorageGroupProcessor storageGroupProcessor;
- private String systemDir = TestConstant.BASE_OUTPUT_PATH.concat("data")
- .concat(File.separator).concat("info");
-
- private int prevUnseqLevelNum = 0;
-
- @Before
- public void setUp() throws Exception {
- prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum();
- IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2);
- EnvironmentUtils.envSetUp();
- MetadataManagerHelper.initMetadata();
- storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup,
- new DirectFlushPolicy());
- insertData();
- }
-
- @After
- public void tearDown() throws Exception {
- FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
- storageGroupProcessor.syncDeleteDataFiles();
- EnvironmentUtils.cleanEnv();
- EnvironmentUtils.cleanDir(systemDir);
- IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum);
- }
-
- private void insertOneRecord(long time, int num)
- throws WriteProcessException, IllegalPathException {
- TSRecord record = new TSRecord(time, storageGroup);
- record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId0, String.valueOf(num)));
- record.addTuple(DataPoint.getDataPoint(TSDataType.INT64, measurementId1, String.valueOf(num)));
- record.addTuple(DataPoint.getDataPoint(TSDataType.FLOAT, measurementId2, String.valueOf(num)));
- record.addTuple(DataPoint.getDataPoint(TSDataType.DOUBLE, measurementId3, String.valueOf(num)));
- record.addTuple(DataPoint.getDataPoint(TSDataType.BOOLEAN, measurementId4, "True"));
- InsertRowPlan insertRowPlan = new InsertRowPlan(record);
- storageGroupProcessor.insert(insertRowPlan);
- }
-
- protected void insertData() throws IOException, WriteProcessException, IllegalPathException {
- for (int j = 1; j <= 100; j++) {
- insertOneRecord(j, j);
- }
- for (TsFileProcessor tsFileProcessor : storageGroupProcessor
- .getWorkSequenceTsFileProcessors()) {
- tsFileProcessor.syncFlush();
- }
-
- for (int j = 10; j >= 1; j--) {
- insertOneRecord(j, j);
- }
- for (int j = 11; j <= 20; j++) {
- insertOneRecord(j, j);
- }
- storageGroupProcessor.syncCloseAllWorkingTsFileProcessors();
-
- for (int j = 21; j <= 30; j += 2) {
- insertOneRecord(j, 0); // will be covered when read
- }
- storageGroupProcessor.syncCloseAllWorkingTsFileProcessors();
-
- for (int j = 21; j <= 30; j += 2) {
- insertOneRecord(j, j);
- }
- storageGroupProcessor.syncCloseAllWorkingTsFileProcessors();
-
- insertOneRecord(2, 100);
- }
-
- @Test
- public void test1() throws IOException, QueryProcessException, IllegalPathException {
- IoTDBDescriptor.getInstance().getConfig().setMetaDataCacheEnable(false);
- QueryDataSource queryDataSource = storageGroupProcessor
- .query(new PartialPath(storageGroup), measurementId5, context, null, null);
-
- List<TsFileResource> seqResources = queryDataSource.getSeqResources();
- List<TsFileResource> unseqResources = queryDataSource.getUnseqResources();
-
- Assert.assertEquals(1, seqResources.size());
- Assert.assertEquals(3, unseqResources.size());
- Assert.assertTrue(seqResources.get(0).isClosed());
- Assert.assertTrue(unseqResources.get(0).isClosed());
- Assert.assertTrue(unseqResources.get(1).isClosed());
- Assert.assertTrue(unseqResources.get(2).isClosed());
-
- List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
- .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null);
- Assert.assertEquals(0, metaDataList.size());
- }
-
- @Test
- public void test2() throws IOException, QueryProcessException, IllegalPathException {
- IoTDBDescriptor.getInstance().getConfig().setMetaDataCacheEnable(true);
- QueryDataSource queryDataSource = storageGroupProcessor
- .query(new PartialPath(storageGroup), measurementId5, context, null, null);
-
- List<TsFileResource> seqResources = queryDataSource.getSeqResources();
- List<TsFileResource> unseqResources = queryDataSource.getUnseqResources();
-
- Assert.assertEquals(1, seqResources.size());
- Assert.assertEquals(3, unseqResources.size());
- Assert.assertTrue(seqResources.get(0).isClosed());
- Assert.assertTrue(unseqResources.get(0).isClosed());
- Assert.assertTrue(unseqResources.get(1).isClosed());
- Assert.assertTrue(unseqResources.get(2).isClosed());
-
- List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
- .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null);
- Assert.assertEquals(0, metaDataList.size());
- }
-
-
-}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
index 7fbc509..1747971 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
@@ -30,9 +30,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
-import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -91,7 +89,6 @@ abstract class LevelCompactionTest {
IoTDBDescriptor.getInstance().getConfig()
.setMergeChunkPointNumberThreshold(prevMergeChunkThreshold);
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
IoTDB.metaManager.clear();
EnvironmentUtils.cleanAllDir();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index e296014..6746aec 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -90,7 +89,6 @@ abstract class MergeTest {
unseqResources.clear();
IoTDBDescriptor.getInstance().getConfig().setMergeChunkPointNumberThreshold(prevMergeChunkThreshold);
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
IoTDB.metaManager.clear();
EnvironmentUtils.cleanAllDir();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java
index 74a20dc..578f298a 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java
@@ -28,7 +28,6 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
@@ -157,13 +156,11 @@ public class IoTDBClearCacheIT {
Assert.assertEquals(10, cnt);
}
assertFalse(ChunkCache.getInstance().isEmpty());
- assertFalse(ChunkMetadataCache.getInstance().isEmpty());
assertFalse(TimeSeriesMetadataCache.getInstance().isEmpty());
statement.execute("CLEAR CACHE");
assertTrue(ChunkCache.getInstance().isEmpty());
- assertTrue(ChunkMetadataCache.getInstance().isEmpty());
assertTrue(TimeSeriesMetadataCache.getInstance().isEmpty());
} catch (Exception e) {
diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
index 6bff6c1..b86e766 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
@@ -30,7 +30,6 @@ import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -74,7 +73,6 @@ public class SeriesReaderTestUtil {
seqResources.clear();
unseqResources.clear();
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
IoTDB.metaManager.clear();
EnvironmentUtils.cleanAllDir();
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 2dafafd..4420922 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -37,7 +37,6 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.compaction.CompactionMergeTaskPoolManager;
import org.apache.iotdb.db.exception.StorageEngineException;
@@ -128,7 +127,6 @@ public class EnvironmentUtils {
// clean cache
if (config.isMetaDataCacheEnable()) {
ChunkCache.getInstance().clear();
- ChunkMetadataCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
}
// close metadata
diff --git a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
index ed458f2..90c596e 100644
--- a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
+++ b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
@@ -18,6 +18,13 @@
*/
package org.apache.iotdb.spark.db;
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer;
@@ -26,7 +33,8 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.StorageEngineException;
@@ -42,14 +50,6 @@ import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Locale;
-
/**
* <p>
* This class is used for cleaning test environment in unit test and integration test
@@ -120,7 +120,8 @@ public class EnvironmentUtils {
MultiFileLogNodeManager.getInstance().stop();
// clean cache
if (config.isMetaDataCacheEnable()) {
- ChunkMetadataCache.getInstance().clear();
+ ChunkCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
}
// close metadata
IoTDB.metaManager.clear();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 083c9e1..981a764 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -76,6 +76,9 @@ public class ChunkMetadata implements Accountable {
// used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata
private boolean isSeq = true;
+ private boolean isClosed;
+ private String filePath;
+
private ChunkMetadata() {
}
@@ -289,4 +292,20 @@ public class ChunkMetadata implements Accountable {
public boolean isSeq() {
return isSeq;
}
+
+ public boolean isClosed() {
+ return isClosed;
+ }
+
+ public void setClosed(boolean closed) {
+ isClosed = closed;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
}
\ No newline at end of file
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index a8c30dd..13ff2a0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -58,8 +58,7 @@ public class TimeseriesMetadata implements Accountable {
// used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata
private boolean isSeq = true;
- private ByteBuffer chunkMetadataListByteBuffer;
-
+ // used to save chunk metadata list while serializing
private PublicBAOS chunkMetadataListBuffer;
private ArrayList<ChunkMetadata> chunkMetadataList;
@@ -87,9 +86,10 @@ public class TimeseriesMetadata implements Accountable {
this.dataType = timeseriesMetadata.dataType;
this.statistics = timeseriesMetadata.statistics;
this.modified = timeseriesMetadata.modified;
+ this.chunkMetadataList = new ArrayList<>(timeseriesMetadata.chunkMetadataList);
}
- public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer) {
+ public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata) {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer));
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
@@ -99,8 +99,16 @@ public class TimeseriesMetadata implements Accountable {
timeseriesMetaData
.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
timeseriesMetaData.setStatistics(Statistics.deserialize(buffer, timeseriesMetaData.dataType));
- timeseriesMetaData.chunkMetadataListByteBuffer = buffer.slice();
- timeseriesMetaData.chunkMetadataListByteBuffer.limit(chunkMetaDataListDataSize);
+ if (needChunkMetadata) {
+ ByteBuffer byteBuffer = buffer.slice();
+ byteBuffer.limit(chunkMetaDataListDataSize);
+ timeseriesMetaData.chunkMetadataList = new ArrayList<>();
+ while (byteBuffer.hasRemaining()) {
+ timeseriesMetaData.chunkMetadataList.add(ChunkMetadata.deserializeFrom(byteBuffer, timeseriesMetaData));
+ }
+ // minimize the storage of an ArrayList instance.
+ timeseriesMetaData.chunkMetadataList.trimToSize();
+ }
buffer.position(buffer.position() + chunkMetaDataListDataSize);
return timeseriesMetaData;
}
@@ -183,15 +191,6 @@ public class TimeseriesMetadata implements Accountable {
}
public List<ChunkMetadata> getChunkMetadataList() {
- if (chunkMetadataList == null) {
- chunkMetadataList = new ArrayList<>();
- while (chunkMetadataListByteBuffer.hasRemaining()) {
- chunkMetadataList.add(ChunkMetadata.deserializeFrom(chunkMetadataListByteBuffer, this));
- }
- // minimize the storage of an ArrayList instance.
- chunkMetadataList.trimToSize();
- chunkMetadataListByteBuffer = null;
- }
return chunkMetadataList;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 1661423..e059751 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -308,7 +308,8 @@ public class TsFileSequenceReader implements AutoCloseable {
private Map<String, TimeseriesMetadata> readDeviceMetadataFromDisk(String device)
throws IOException {
readFileMetadata();
- List<TimeseriesMetadata> timeseriesMetadataList = getDeviceTimeseriesMetadata(device);
+ List<TimeseriesMetadata> timeseriesMetadataList = getDeviceTimeseriesMetadataWithoutChunkMetadata(
+ device);
Map<String, TimeseriesMetadata> deviceMetadata = new HashMap<>();
for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) {
deviceMetadata.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata);
@@ -343,7 +344,7 @@ public class TsFileSequenceReader implements AutoCloseable {
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
while (buffer.hasRemaining()) {
try {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
} catch (BufferOverflowException e) {
logger.error("Something error happened while deserializing TimeseriesMetadata of file {}",
file);
@@ -389,7 +390,7 @@ public class TsFileSequenceReader implements AutoCloseable {
while (buffer.hasRemaining()) {
TimeseriesMetadata timeseriesMetadata;
try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
+ timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
} catch (BufferOverflowException e) {
logger.error("Something error happened while deserializing TimeseriesMetadata of file {}",
file);
@@ -439,7 +440,7 @@ public class TsFileSequenceReader implements AutoCloseable {
measurementMetadataIndexPair.right);
while (buffer.hasRemaining()) {
try {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
} catch (BufferOverflowException e) {
logger.error("Something error happened while deserializing TimeseriesMetadata of file {}",
file);
@@ -463,51 +464,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return resultTimeseriesMetadataList;
}
- /**
- * Traverse and read TimeseriesMetadata of specific measurements in one device. This method need
- * to deserialize all TimeseriesMetadata and then judge, in order to avoid frequent I/O when the
- * number of queried measurements is too large. Attention: This method is not used currently
- *
- * @param timeseriesMetadataList TimeseriesMetadata list, to store the result
- * @param type MetadataIndexNode type
- * @param metadataIndexPair <MetadataIndexEntry, offset> pair
- * @param measurements measurements to be queried
- * @throws IOException io error
- */
- private void traverseAndReadTimeseriesMetadataInOneDevice(
- List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNodeType type,
- Pair<MetadataIndexEntry, Long> metadataIndexPair, Set<String> measurements)
- throws IOException {
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- switch (type) {
- case LEAF_DEVICE:
- case INTERNAL_MEASUREMENT:
- MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
- int metadataIndexListSize = metadataIndexNode.getChildren().size();
- for (int i = 0; i < metadataIndexListSize; i++) {
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexListSize - 1) {
- endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
- }
- traverseAndReadTimeseriesMetadataInOneDevice(timeseriesMetadataList,
- metadataIndexNode.getNodeType(),
- new Pair<>(metadataIndexNode.getChildren().get(i), endOffset), measurements);
- }
- break;
- case LEAF_MEASUREMENT:
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
- if (measurements.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
- }
- }
- break;
- default:
- throw new IOException("Failed to traverse and read TimeseriesMetadata in device: " +
- metadataIndexPair.left.getName() + ". Wrong MetadataIndexEntry type.");
- }
- }
-
protected int binarySearchInTimeseriesMetadataList(
List<TimeseriesMetadata> timeseriesMetadataList, String key) {
int low = 0;
@@ -577,35 +533,15 @@ public class TsFileSequenceReader implements AutoCloseable {
public Map<String, List<ChunkMetadata>> readChunkMetadataInDevice(String device)
throws IOException {
readFileMetadata();
-
- long start = 0;
- int size = 0;
List<TimeseriesMetadata> timeseriesMetadataMap = getDeviceTimeseriesMetadata(device);
if (timeseriesMetadataMap.isEmpty()) {
return new HashMap<>();
}
- for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap) {
- if (start == 0) {
- start = timeseriesMetadata.getOffsetOfChunkMetaDataList();
- }
- size += timeseriesMetadata.getDataSizeOfChunkMetaDataList();
- }
- // read buffer of all ChunkMetadatas of this device
- ByteBuffer buffer = readData(start, size);
Map<String, List<ChunkMetadata>> seriesMetadata = new HashMap<>();
- int index = 0;
- int curSize = timeseriesMetadataMap.get(index).getDataSizeOfChunkMetaDataList();
- while (buffer.hasRemaining()) {
- if (buffer.position() >= curSize) {
- index++;
- curSize += timeseriesMetadataMap.get(index).getDataSizeOfChunkMetaDataList();
- }
- ChunkMetadata chunkMetadata = ChunkMetadata
- .deserializeFrom(buffer, timeseriesMetadataMap.get(index));
- seriesMetadata.computeIfAbsent(chunkMetadata.getMeasurementUid(), key -> new ArrayList<>())
- .add(chunkMetadata);
+ for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap) {
+ seriesMetadata
+ .put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata.getChunkMetadataList());
}
-
return seriesMetadata;
}
@@ -633,15 +569,17 @@ public class TsFileSequenceReader implements AutoCloseable {
* @param buffer byte buffer
* @param deviceId String
* @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list
+ * @param needChunkMetadata deserialize chunk metadata list or not
*/
private void generateMetadataIndex(MetadataIndexEntry metadataIndex, ByteBuffer buffer,
String deviceId, MetadataIndexNodeType type,
- Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap) throws IOException {
+ Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap, boolean needChunkMetadata)
+ throws IOException {
try {
if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
while (buffer.hasRemaining()) {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata));
}
timeseriesMetadataMap.computeIfAbsent(deviceId, k -> new ArrayList<>())
.addAll(timeseriesMetadataList);
@@ -657,7 +595,7 @@ public class TsFileSequenceReader implements AutoCloseable {
ByteBuffer nextBuffer = readData(metadataIndexNode.getChildren().get(i).getOffset(),
endOffset);
generateMetadataIndex(metadataIndexNode.getChildren().get(i), nextBuffer, deviceId,
- metadataIndexNode.getNodeType(), timeseriesMetadataMap);
+ metadataIndexNode.getNodeType(), timeseriesMetadataMap, needChunkMetadata);
}
}
} catch (BufferOverflowException e) {
@@ -666,6 +604,9 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
+ /**
+ * TimeseriesMetadata don't need deserialize chunk metadata list
+ */
public Map<String, List<TimeseriesMetadata>> getAllTimeseriesMetadata() throws IOException {
if (tsFileMetaData == null) {
readFileMetadata();
@@ -681,11 +622,37 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
generateMetadataIndex(metadataIndexEntry, buffer, null,
- metadataIndexNode.getNodeType(), timeseriesMetadataMap);
+ metadataIndexNode.getNodeType(), timeseriesMetadataMap, false);
}
return timeseriesMetadataMap;
}
+ /**
+ * This method will only deserialize the TimeseriesMetadata, not including chunk metadata list
+ */
+ private List<TimeseriesMetadata> getDeviceTimeseriesMetadataWithoutChunkMetadata(String device)
+ throws IOException {
+ MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
+ Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
+ metadataIndexNode, device, true, true);
+ if (metadataIndexPair == null) {
+ return Collections.emptyList();
+ }
+ ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
+ generateMetadataIndex(metadataIndexPair.left, buffer, device,
+ MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap, false);
+ List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>();
+ for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) {
+ deviceTimeseriesMetadata.addAll(timeseriesMetadataList);
+ }
+ return deviceTimeseriesMetadata;
+ }
+
+ /**
+ * This method will not only deserialize the TimeseriesMetadata, but also all the chunk metadata
+ * list meanwhile.
+ */
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
@@ -696,7 +663,7 @@ public class TsFileSequenceReader implements AutoCloseable {
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
generateMetadataIndex(metadataIndexPair.left, buffer, device,
- MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap);
+ MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap, true);
List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>();
for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) {
deviceTimeseriesMetadata.addAll(timeseriesMetadataList);
@@ -729,7 +696,8 @@ public class TsFileSequenceReader implements AutoCloseable {
Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex
.getChildIndexEntry(name, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
- return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, isDeviceLevel,
+ return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name,
+ isDeviceLevel,
false);
}
} catch (BufferOverflowException e) {
@@ -1145,24 +1113,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public List<ChunkMetadata> readChunkMetaDataList(TimeseriesMetadata timeseriesMetaData)
throws IOException {
- try {
- readFileMetadata();
- ArrayList<ChunkMetadata> chunkMetadataList = new ArrayList<>();
- long startOffsetOfChunkMetadataList = timeseriesMetaData.getOffsetOfChunkMetaDataList();
- int dataSizeOfChunkMetadataList = timeseriesMetaData.getDataSizeOfChunkMetaDataList();
-
- ByteBuffer buffer = readData(startOffsetOfChunkMetadataList, dataSizeOfChunkMetadataList);
- while (buffer.hasRemaining()) {
- chunkMetadataList.add(ChunkMetadata.deserializeFrom(buffer, timeseriesMetaData));
- }
-
- // minimize the storage of an ArrayList instance.
- chunkMetadataList.trimToSize();
- return chunkMetadataList;
- } catch (BufferOverflowException e) {
- logger.error("Something error happened while reading ChunkMetaDataList of file {}", file);
- throw e;
- }
+ return timeseriesMetaData.getChunkMetadataList();
}
/**
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index bbda4b8..95da50f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -31,12 +31,4 @@ public interface IChunkMetadataLoader {
*/
List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
throws IOException;
-
- /**
- * For query 0.9/v1 tsfile only
- * @param chunkMetadataList
- * @throws IOException
- */
- void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) throws IOException;
-
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java
index b5ba185..05bdc3e 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java
@@ -67,7 +67,7 @@ public class TimeSeriesMetadataTest {
ByteBuffer buffer = ByteBuffer.allocate((int) fch.size());
fch.read(buffer);
buffer.flip();
- metaData = TimeseriesMetadata.deserializeFrom(buffer);
+ metaData = TimeseriesMetadata.deserializeFrom(buffer, true);
return metaData;
} catch (IOException e) {
e.printStackTrace();