You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/02/14 04:16:16 UTC
[incubator-iotdb] 01/02: add cached chunk in server
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch CachedChunk
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a74ce816788ae1424e4aee18a69f5d77d3c1da06
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Fri Feb 14 11:24:41 2020 +0800
add cached chunk in server
---
.../resources/conf/iotdb-engine.properties | 6 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 25 +++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 17 ++-
.../db/engine/cache/CacheHitRatioMonitor.java | 5 +
.../engine/cache/CacheHitRatioMonitorMXBean.java | 2 +
.../apache/iotdb/db/engine/cache/ChunkCache.java | 153 +++++++++++++++++++++
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 13 +-
.../iotdb/db/query/control/FileReaderManager.java | 15 --
.../db/query/reader/chunk/DiskChunkLoader.java | 43 ++----
.../iotdb/db/query/reader/series/SeriesReader.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
.../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 4 +-
...kLoaderImpl.java => CachedChunkLoaderImpl.java} | 6 +-
.../tsfile/read/controller/ChunkLoaderTest.java | 2 +-
.../read/query/executor/QueryExecutorTest.java | 4 +-
.../query/timegenerator/ReaderByTimestampTest.java | 4 +-
.../query/timegenerator/TimeGeneratorTest.java | 4 +-
.../iotdb/tsfile/read/reader/ReaderTest.java | 6 +-
18 files changed, 224 insertions(+), 93 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 64a1185..59b8172 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -275,9 +275,9 @@ chunk_merge_point_threshold=20480
# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
meta_data_cache_enable=true
-# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query.
-# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10
-filemeta_chunkmeta_free_memory_proportion=3:6:10
+# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, ChunkCache and Free Memory Used in Query.
+# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 3:6:10:20
+filemeta_chunkmeta_free_memory_proportion=3:6:10:20
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index cedbec2..251c11c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -188,12 +188,17 @@ public class IoTDBConfig {
/**
* Memory allocated for fileMetaData cache in read process
*/
- private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19;
+ private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 39;
/**
* Memory allocated for chunkMetaData cache in read process
*/
- private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19;
+ private long allocateMemoryForChunkMetaDataCache = allocateMemoryForRead * 6 / 39;
+
+ /**
+ * Memory allocated for chunk cache in read process
+ */
+ private long allocateMemoryForChunkCache = allocateMemoryForRead * 10 / 39;
/**
* The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default
@@ -1047,12 +1052,20 @@ public class IoTDBConfig {
this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache;
}
- public long getAllocateMemoryForChumkMetaDataCache() {
- return allocateMemoryForChumkMetaDataCache;
+ public long getAllocateMemoryForChunkMetaDataCache() {
+ return allocateMemoryForChunkMetaDataCache;
+ }
+
+ public void setAllocateMemoryForChunkMetaDataCache(long allocateMemoryForChunkMetaDataCache) {
+ this.allocateMemoryForChunkMetaDataCache = allocateMemoryForChunkMetaDataCache;
+ }
+
+ public long getAllocateMemoryForChunkCache() {
+ return allocateMemoryForChunkCache;
}
- public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) {
- this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache;
+ public void setAllocateMemoryForChunkCache(long allocateMemoryForChunkCache) {
+ this.allocateMemoryForChunkCache = allocateMemoryForChunkCache;
}
public boolean isEnableWatermark() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index e5b07bc..b10a96e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,14 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.time.ZoneId;
-import java.util.Properties;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.utils.FilePathUtils;
@@ -33,6 +25,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.net.URL;
+import java.time.ZoneId;
+import java.util.Properties;
+
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
@@ -527,8 +524,10 @@ public class IoTDBDescriptor {
try {
conf.setAllocateMemoryForFileMetaDataCache(
maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
- conf.setAllocateMemoryForChumkMetaDataCache(
+ conf.setAllocateMemoryForChunkMetaDataCache(
maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
+ conf.setAllocateMemoryForChunkCache(
+ maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum);
} catch (Exception e) {
throw new RuntimeException(
"Each subsection of configuration item filemeta_chunkmeta_free_memory_proportion should be an integer, which is "
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
index 09b0419..0d06b9b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
@@ -66,6 +66,11 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
return tsfileMetaDataHitRatio;
}
+ @Override
+ public double getChunkHitRatio() {
+ return ChunkCache.getInstance().calculateChunkHitRatio();
+ }
+
public static CacheHitRatioMonitor getInstance() {
return instance;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
index 9cd10c5..5bc02e7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
@@ -23,4 +23,6 @@ public interface CacheHitRatioMonitorMXBean {
double getChunkMetaDataHitRatio();
double getTsfileMetaDataHitRatio();
+
+ double getChunkHitRatio();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
new file mode 100644
index 0000000..cfb9d63
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This class is used to cache <code>Chunk</code> of <code>ChunkMetaData</code> in IoTDB. The caching
+ * strategy is LRU.
+ */
+public class ChunkCache {
+
+ private static final Logger logger = LoggerFactory.getLogger(ChunkCache.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE = config.getAllocateMemoryForChunkCache();
+ private static boolean cacheEnable = config.isMetaDataCacheEnable();
+
+ private final LRULinkedHashMap<ChunkMetaData, Chunk> lruCache;
+
+ private AtomicLong cacheHitNum = new AtomicLong();
+ private AtomicLong cacheRequestNum = new AtomicLong();
+
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+ private final Lock readLock = lock.readLock();
+ private final Lock writeLock = lock.writeLock();
+
+
+ private ChunkCache() {
+ lruCache = new LRULinkedHashMap<ChunkMetaData, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE, true) {
+ @Override
+ protected long calEntrySize(ChunkMetaData key, Chunk value) {
+ return RamUsageEstimator.sizeOf(value) + RamUsageEstimator.sizeOf(value);
+ }
+ };
+ }
+
+ public static ChunkCache getInstance() {
+ return ChunkCacheHolder.INSTANCE;
+ }
+
+ public Chunk get(ChunkMetaData chunkMetaData, TsFileSequenceReader reader) throws IOException {
+ if (!cacheEnable) {
+ Chunk chunk = reader.readMemChunk(chunkMetaData);
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+
+ cacheRequestNum.incrementAndGet();
+
+ try {
+ readLock.lock();
+ if (lruCache.containsKey(chunkMetaData)) {
+ cacheHitNum.incrementAndGet();
+ printCacheLog(true);
+ Chunk chunk = lruCache.get(chunkMetaData);
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ writeLock.lock();
+ if (lruCache.containsKey(chunkMetaData)) {
+ writeLock.unlock();
+ readLock.lock();
+ cacheHitNum.incrementAndGet();
+ printCacheLog(true);
+ Chunk chunk = lruCache.get(chunkMetaData);
+ readLock.unlock();
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+ printCacheLog(false);
+ Chunk chunk = reader.readMemChunk(chunkMetaData);
+ lruCache.put(chunkMetaData, chunk);
+ writeLock.unlock();
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+
+ }
+
+ private void printCacheLog(boolean isHit) {
+ if (!logger.isDebugEnabled()) {
+ return;
+ }
+ logger.debug(
+ "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.",
+ isHit ? "" : "didn't ", cacheRequestNum.get(),
+ cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ }
+
+ public double calculateChunkHitRatio() {
+ if (cacheRequestNum.get() != 0) {
+ return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
+ } else {
+ return 0;
+ }
+ }
+
+
+ /**
+ * clear LRUCache.
+ */
+ public void clear() {
+ writeLock.lock();
+ if (lruCache != null) {
+ lruCache.clear();
+ }
+ writeLock.unlock();
+ }
+
+ public void remove(ChunkMetaData chunkMetaData) {
+ writeLock.lock();
+ if (chunkMetaData != null) {
+ lruCache.remove(chunkMetaData);
+ }
+ writeLock.unlock();
+ }
+
+ /**
+ * singleton pattern.
+ */
+ private static class ChunkCacheHolder {
+
+ private static final ChunkCache INSTANCE = new ChunkCache();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index e5b749f..6ae882a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -18,13 +18,6 @@
*/
package org.apache.iotdb.db.engine.cache;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
@@ -37,6 +30,10 @@ import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching
* strategy is LRU.
@@ -45,7 +42,7 @@ public class DeviceMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache();
+ private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChunkMetaDataCache();
private static StorageEngine storageEngine = StorageEngine.getInstance();
private static boolean cacheEnable = config.isMetaDataCacheEnable();
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index 850d444..8c86de2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -25,13 +25,10 @@ import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.UnClosedTsFileReader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
@@ -73,8 +70,6 @@ public class FileReaderManager implements IService {
*/
private Map<TsFileResource, AtomicInteger> unclosedReferenceMap;
- private final Map<TsFileSequenceReader, IChunkLoader> chunkLoaderMap;
-
private ScheduledExecutorService executorService;
private FileReaderManager() {
@@ -82,7 +77,6 @@ public class FileReaderManager implements IService {
unclosedFileReaderMap = new ConcurrentHashMap<>();
closedReferenceMap = new ConcurrentHashMap<>();
unclosedReferenceMap = new ConcurrentHashMap<>();
- chunkLoaderMap = new HashMap<>();
executorService = IoTDBThreadPoolFactory.newScheduledThreadPool(1,
"open-files-manager");
@@ -126,9 +120,6 @@ public class FileReaderManager implements IService {
if (refAtom != null && refAtom.get() == 0) {
try {
- synchronized (chunkLoaderMap) {
- chunkLoaderMap.remove(reader);
- }
reader.close();
} catch (IOException e) {
logger.error("Can not close TsFileSequenceReader {} !", reader.getFileName(), e);
@@ -171,12 +162,6 @@ public class FileReaderManager implements IService {
}
- public IChunkLoader get(TsFileSequenceReader reader) {
- synchronized (chunkLoaderMap) {
- return chunkLoaderMap.computeIfAbsent(reader, ChunkLoaderImpl::new);
- }
- }
-
/**
* Increase the reference count of the reader specified by filePath. Only when the reference count
* of a reader equals zero, the reader can be closed and removed.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
similarity index 51%
copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
copy to server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index a1935d5..7206f6e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -16,51 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.read.controller;
-import org.apache.iotdb.tsfile.common.cache.LRUCache;
+package org.apache.iotdb.db.query.reader.chunk;
+
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import java.io.IOException;
-/**
- * Read one Chunk and cache it into a LRUCache.
- */
-public class ChunkLoaderImpl implements IChunkLoader {
-
- private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000;
- private TsFileSequenceReader reader;
- private LRUCache<ChunkMetaData, Chunk> chunkCache;
-
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
- this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE);
- }
-
- /**
- * constructor of ChunkLoaderImpl.
- *
- * @param fileSequenceReader file sequence reader
- * @param cacheSize cache size
- */
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
-
- this.reader = fileSequenceReader;
+public class DiskChunkLoader implements IChunkLoader {
- chunkCache = new LRUCache<ChunkMetaData, Chunk>(cacheSize) {
+ private final TsFileSequenceReader reader;
- @Override
- public Chunk loadObjectByKey(ChunkMetaData metaData) throws IOException {
- return reader.readMemChunk(metaData);
- }
- };
+ public DiskChunkLoader(TsFileSequenceReader reader) {
+ this.reader = reader;
}
@Override
public Chunk getChunk(ChunkMetaData chunkMetaData) throws IOException {
- Chunk chunk = chunkCache.get(chunkMetaData);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ return ChunkCache.getInstance().get(chunkMetaData, reader);
}
@Override
@@ -70,6 +47,6 @@ public class ChunkLoaderImpl implements IChunkLoader {
@Override
public void clear() {
- chunkCache.clear();
+ // no cache need clear
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 5140dbb..0158cfc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
import org.apache.iotdb.db.query.reader.chunk.MemChunkLoader;
import org.apache.iotdb.db.query.reader.chunk.MemChunkReader;
import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
@@ -38,7 +39,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
@@ -366,7 +366,7 @@ public class SeriesReader {
if (data.getChunkLoader() == null) {
TsFileSequenceReader tsFileSequenceReader = FileReaderManager.getInstance()
.get(resource, resource.isClosed());
- data.setChunkLoader(new ChunkLoaderImpl(tsFileSequenceReader));
+ data.setChunkLoader(new DiskChunkLoader(tsFileSequenceReader));
}
}
List<ReadOnlyMemChunk> memChunks = resource.getReadOnlyMemChunk();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index f961299..6fe01ff 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -39,7 +39,7 @@ import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
@@ -161,7 +161,7 @@ public class UnseqTsFileRecoverTest {
TsFileSequenceReader fileReader = new TsFileSequenceReader(tsF.getPath(), true);
IMetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(fileReader);
- IChunkLoader chunkLoader = new ChunkLoaderImpl(fileReader);
+ IChunkLoader chunkLoader = new CachedChunkLoaderImpl(fileReader);
Path path = new Path("device1", "sensor1");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
index 89b64db..c4e9fbb 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
@@ -20,7 +20,7 @@ package org.apache.iotdb.tsfile.read;
import java.io.IOException;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
@@ -40,7 +40,7 @@ public class ReadOnlyTsFile implements AutoCloseable {
public ReadOnlyTsFile(TsFileSequenceReader fileReader) throws IOException {
this.fileReader = fileReader;
this.metadataQuerier = new MetadataQuerierByFileImpl(fileReader);
- this.chunkLoader = new ChunkLoaderImpl(fileReader);
+ this.chunkLoader = new CachedChunkLoaderImpl(fileReader);
tsFileExecutor = new TsFileExecutor(metadataQuerier, chunkLoader);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
similarity index 90%
rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
index a1935d5..fbe7d92 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
@@ -28,13 +28,13 @@ import java.io.IOException;
/**
* Read one Chunk and cache it into a LRUCache.
*/
-public class ChunkLoaderImpl implements IChunkLoader {
+public class CachedChunkLoaderImpl implements IChunkLoader {
private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000;
private TsFileSequenceReader reader;
private LRUCache<ChunkMetaData, Chunk> chunkCache;
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
+ public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE);
}
@@ -44,7 +44,7 @@ public class ChunkLoaderImpl implements IChunkLoader {
* @param fileSequenceReader file sequence reader
* @param cacheSize cache size
*/
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
+ public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
this.reader = fileSequenceReader;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
index 19a8416..5094bd2 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
@@ -54,7 +54,7 @@ public class ChunkLoaderTest {
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d2.s1"));
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
for (ChunkMetaData chunkMetaData : chunkMetaDataList) {
Chunk chunk = seriesChunkLoader.getChunk(chunkMetaData);
ChunkHeader chunkHeader = chunk.getHeader();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
index 54eacc0..28d772c 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
@@ -59,7 +59,7 @@ public class QueryExecutorTest {
TsFileGeneratorForTest.generateFile(rowCount, 16 * 1024 * 1024, 10000);
fileReader = new TsFileSequenceReader(FILE_PATH);
metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader);
- chunkLoader = new ChunkLoaderImpl(fileReader);
+ chunkLoader = new CachedChunkLoaderImpl(fileReader);
queryExecutorWithQueryFilter = new TsFileExecutor(metadataQuerierByFile, chunkLoader);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
index e5818eb..3045dc8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.series.AbstractFileSeriesReader;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
@@ -61,7 +61,7 @@ public class ReaderByTimestampTest {
@Test
public void readByTimestamp() throws IOException {
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));
AbstractFileSeriesReader seriesReader = new FileSeriesReader(seriesChunkLoader,
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
index e1b9687..38bc71d 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
@@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression;
@@ -53,7 +53,7 @@ public class TimeGeneratorTest {
TsFileGeneratorForTest.generateFile(1000, 10 * 1024 * 1024, 10000);
fileReader = new TsFileSequenceReader(FILE_PATH);
metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader);
- chunkLoader = new ChunkLoaderImpl(fileReader);
+ chunkLoader = new CachedChunkLoaderImpl(fileReader);
}
@After
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
index 93c72dc..2622ca8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
import org.apache.iotdb.tsfile.read.filter.TimeFilter;
@@ -65,7 +65,7 @@ public class ReaderTest {
@Test
public void readTest() throws IOException {
int count = 0;
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));
@@ -101,7 +101,7 @@ public class ReaderTest {
@Test
public void readWithFilterTest() throws IOException {
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));