You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/02/14 04:16:15 UTC
[incubator-iotdb] branch CachedChunk created (now be7abc7)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch CachedChunk
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at be7abc7 Merge branch 'new_series_reader' into CachedChunk
This branch includes the following new commits:
new a74ce81 add cached chunk in server
new be7abc7 Merge branch 'new_series_reader' into CachedChunk
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/02: add cached chunk in server
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch CachedChunk
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a74ce816788ae1424e4aee18a69f5d77d3c1da06
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Fri Feb 14 11:24:41 2020 +0800
add cached chunk in server
---
.../resources/conf/iotdb-engine.properties | 6 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 25 +++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 17 ++-
.../db/engine/cache/CacheHitRatioMonitor.java | 5 +
.../engine/cache/CacheHitRatioMonitorMXBean.java | 2 +
.../apache/iotdb/db/engine/cache/ChunkCache.java | 153 +++++++++++++++++++++
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 13 +-
.../iotdb/db/query/control/FileReaderManager.java | 15 --
.../db/query/reader/chunk/DiskChunkLoader.java | 43 ++----
.../iotdb/db/query/reader/series/SeriesReader.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
.../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 4 +-
...kLoaderImpl.java => CachedChunkLoaderImpl.java} | 6 +-
.../tsfile/read/controller/ChunkLoaderTest.java | 2 +-
.../read/query/executor/QueryExecutorTest.java | 4 +-
.../query/timegenerator/ReaderByTimestampTest.java | 4 +-
.../query/timegenerator/TimeGeneratorTest.java | 4 +-
.../iotdb/tsfile/read/reader/ReaderTest.java | 6 +-
18 files changed, 224 insertions(+), 93 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 64a1185..59b8172 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -275,9 +275,9 @@ chunk_merge_point_threshold=20480
# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
meta_data_cache_enable=true
-# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query.
-# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10
-filemeta_chunkmeta_free_memory_proportion=3:6:10
+# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, ChunkCache and Free Memory Used in Query.
+# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 3:6:10:20
+filemeta_chunkmeta_free_memory_proportion=3:6:10:20
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index cedbec2..251c11c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -188,12 +188,17 @@ public class IoTDBConfig {
/**
* Memory allocated for fileMetaData cache in read process
*/
- private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19;
+ private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 39;
/**
* Memory allocated for chunkMetaData cache in read process
*/
- private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19;
+ private long allocateMemoryForChunkMetaDataCache = allocateMemoryForRead * 6 / 39;
+
+ /**
+ * Memory allocated for chunk cache in read process
+ */
+ private long allocateMemoryForChunkCache = allocateMemoryForRead * 10 / 39;
/**
* The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default
@@ -1047,12 +1052,20 @@ public class IoTDBConfig {
this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache;
}
- public long getAllocateMemoryForChumkMetaDataCache() {
- return allocateMemoryForChumkMetaDataCache;
+ public long getAllocateMemoryForChunkMetaDataCache() {
+ return allocateMemoryForChunkMetaDataCache;
+ }
+
+ public void setAllocateMemoryForChunkMetaDataCache(long allocateMemoryForChunkMetaDataCache) {
+ this.allocateMemoryForChunkMetaDataCache = allocateMemoryForChunkMetaDataCache;
+ }
+
+ public long getAllocateMemoryForChunkCache() {
+ return allocateMemoryForChunkCache;
}
- public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) {
- this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache;
+ public void setAllocateMemoryForChunkCache(long allocateMemoryForChunkCache) {
+ this.allocateMemoryForChunkCache = allocateMemoryForChunkCache;
}
public boolean isEnableWatermark() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index e5b07bc..b10a96e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,14 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.time.ZoneId;
-import java.util.Properties;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.utils.FilePathUtils;
@@ -33,6 +25,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.net.URL;
+import java.time.ZoneId;
+import java.util.Properties;
+
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
@@ -527,8 +524,10 @@ public class IoTDBDescriptor {
try {
conf.setAllocateMemoryForFileMetaDataCache(
maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum);
- conf.setAllocateMemoryForChumkMetaDataCache(
+ conf.setAllocateMemoryForChunkMetaDataCache(
maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
+ conf.setAllocateMemoryForChunkCache(
+ maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum);
} catch (Exception e) {
throw new RuntimeException(
"Each subsection of configuration item filemeta_chunkmeta_free_memory_proportion should be an integer, which is "
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
index 09b0419..0d06b9b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
@@ -66,6 +66,11 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
return tsfileMetaDataHitRatio;
}
+ @Override
+ public double getChunkHitRatio() {
+ return ChunkCache.getInstance().calculateChunkHitRatio();
+ }
+
public static CacheHitRatioMonitor getInstance() {
return instance;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
index 9cd10c5..5bc02e7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
@@ -23,4 +23,6 @@ public interface CacheHitRatioMonitorMXBean {
double getChunkMetaDataHitRatio();
double getTsfileMetaDataHitRatio();
+
+ double getChunkHitRatio();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
new file mode 100644
index 0000000..cfb9d63
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This class is used to cache <code>Chunk</code> of <code>ChunkMetaData</code> in IoTDB. The caching
+ * strategy is LRU.
+ */
+public class ChunkCache {
+
+ private static final Logger logger = LoggerFactory.getLogger(ChunkCache.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE = config.getAllocateMemoryForChunkCache();
+ private static boolean cacheEnable = config.isMetaDataCacheEnable();
+
+ private final LRULinkedHashMap<ChunkMetaData, Chunk> lruCache;
+
+ private AtomicLong cacheHitNum = new AtomicLong();
+ private AtomicLong cacheRequestNum = new AtomicLong();
+
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+ private final Lock readLock = lock.readLock();
+ private final Lock writeLock = lock.writeLock();
+
+
+ private ChunkCache() {
+ lruCache = new LRULinkedHashMap<ChunkMetaData, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE, true) {
+ @Override
+ protected long calEntrySize(ChunkMetaData key, Chunk value) {
+ return RamUsageEstimator.sizeOf(value) + RamUsageEstimator.sizeOf(value);
+ }
+ };
+ }
+
+ public static ChunkCache getInstance() {
+ return ChunkCacheHolder.INSTANCE;
+ }
+
+ public Chunk get(ChunkMetaData chunkMetaData, TsFileSequenceReader reader) throws IOException {
+ if (!cacheEnable) {
+ Chunk chunk = reader.readMemChunk(chunkMetaData);
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+
+ cacheRequestNum.incrementAndGet();
+
+ try {
+ readLock.lock();
+ if (lruCache.containsKey(chunkMetaData)) {
+ cacheHitNum.incrementAndGet();
+ printCacheLog(true);
+ Chunk chunk = lruCache.get(chunkMetaData);
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ writeLock.lock();
+ if (lruCache.containsKey(chunkMetaData)) {
+ writeLock.unlock();
+ readLock.lock();
+ cacheHitNum.incrementAndGet();
+ printCacheLog(true);
+ Chunk chunk = lruCache.get(chunkMetaData);
+ readLock.unlock();
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ }
+ printCacheLog(false);
+ Chunk chunk = reader.readMemChunk(chunkMetaData);
+ lruCache.put(chunkMetaData, chunk);
+ writeLock.unlock();
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+
+ }
+
+ private void printCacheLog(boolean isHit) {
+ if (!logger.isDebugEnabled()) {
+ return;
+ }
+ logger.debug(
+ "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.",
+ isHit ? "" : "didn't ", cacheRequestNum.get(),
+ cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ }
+
+ public double calculateChunkHitRatio() {
+ if (cacheRequestNum.get() != 0) {
+ return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
+ } else {
+ return 0;
+ }
+ }
+
+
+ /**
+ * clear LRUCache.
+ */
+ public void clear() {
+ writeLock.lock();
+ if (lruCache != null) {
+ lruCache.clear();
+ }
+ writeLock.unlock();
+ }
+
+ public void remove(ChunkMetaData chunkMetaData) {
+ writeLock.lock();
+ if (chunkMetaData != null) {
+ lruCache.remove(chunkMetaData);
+ }
+ writeLock.unlock();
+ }
+
+ /**
+ * singleton pattern.
+ */
+ private static class ChunkCacheHolder {
+
+ private static final ChunkCache INSTANCE = new ChunkCache();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index e5b749f..6ae882a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -18,13 +18,6 @@
*/
package org.apache.iotdb.db.engine.cache;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
@@ -37,6 +30,10 @@ import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching
* strategy is LRU.
@@ -45,7 +42,7 @@ public class DeviceMetaDataCache {
private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache();
+ private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChunkMetaDataCache();
private static StorageEngine storageEngine = StorageEngine.getInstance();
private static boolean cacheEnable = config.isMetaDataCacheEnable();
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index 850d444..8c86de2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -25,13 +25,10 @@ import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.UnClosedTsFileReader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
@@ -73,8 +70,6 @@ public class FileReaderManager implements IService {
*/
private Map<TsFileResource, AtomicInteger> unclosedReferenceMap;
- private final Map<TsFileSequenceReader, IChunkLoader> chunkLoaderMap;
-
private ScheduledExecutorService executorService;
private FileReaderManager() {
@@ -82,7 +77,6 @@ public class FileReaderManager implements IService {
unclosedFileReaderMap = new ConcurrentHashMap<>();
closedReferenceMap = new ConcurrentHashMap<>();
unclosedReferenceMap = new ConcurrentHashMap<>();
- chunkLoaderMap = new HashMap<>();
executorService = IoTDBThreadPoolFactory.newScheduledThreadPool(1,
"open-files-manager");
@@ -126,9 +120,6 @@ public class FileReaderManager implements IService {
if (refAtom != null && refAtom.get() == 0) {
try {
- synchronized (chunkLoaderMap) {
- chunkLoaderMap.remove(reader);
- }
reader.close();
} catch (IOException e) {
logger.error("Can not close TsFileSequenceReader {} !", reader.getFileName(), e);
@@ -171,12 +162,6 @@ public class FileReaderManager implements IService {
}
- public IChunkLoader get(TsFileSequenceReader reader) {
- synchronized (chunkLoaderMap) {
- return chunkLoaderMap.computeIfAbsent(reader, ChunkLoaderImpl::new);
- }
- }
-
/**
* Increase the reference count of the reader specified by filePath. Only when the reference count
* of a reader equals zero, the reader can be closed and removed.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
similarity index 51%
copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
copy to server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index a1935d5..7206f6e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -16,51 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.read.controller;
-import org.apache.iotdb.tsfile.common.cache.LRUCache;
+package org.apache.iotdb.db.query.reader.chunk;
+
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import java.io.IOException;
-/**
- * Read one Chunk and cache it into a LRUCache.
- */
-public class ChunkLoaderImpl implements IChunkLoader {
-
- private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000;
- private TsFileSequenceReader reader;
- private LRUCache<ChunkMetaData, Chunk> chunkCache;
-
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
- this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE);
- }
-
- /**
- * constructor of ChunkLoaderImpl.
- *
- * @param fileSequenceReader file sequence reader
- * @param cacheSize cache size
- */
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
-
- this.reader = fileSequenceReader;
+public class DiskChunkLoader implements IChunkLoader {
- chunkCache = new LRUCache<ChunkMetaData, Chunk>(cacheSize) {
+ private final TsFileSequenceReader reader;
- @Override
- public Chunk loadObjectByKey(ChunkMetaData metaData) throws IOException {
- return reader.readMemChunk(metaData);
- }
- };
+ public DiskChunkLoader(TsFileSequenceReader reader) {
+ this.reader = reader;
}
@Override
public Chunk getChunk(ChunkMetaData chunkMetaData) throws IOException {
- Chunk chunk = chunkCache.get(chunkMetaData);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType());
+ return ChunkCache.getInstance().get(chunkMetaData, reader);
}
@Override
@@ -70,6 +47,6 @@ public class ChunkLoaderImpl implements IChunkLoader {
@Override
public void clear() {
- chunkCache.clear();
+ // no cache need clear
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 5140dbb..0158cfc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
import org.apache.iotdb.db.query.reader.chunk.MemChunkLoader;
import org.apache.iotdb.db.query.reader.chunk.MemChunkReader;
import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
@@ -38,7 +39,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
@@ -366,7 +366,7 @@ public class SeriesReader {
if (data.getChunkLoader() == null) {
TsFileSequenceReader tsFileSequenceReader = FileReaderManager.getInstance()
.get(resource, resource.isClosed());
- data.setChunkLoader(new ChunkLoaderImpl(tsFileSequenceReader));
+ data.setChunkLoader(new DiskChunkLoader(tsFileSequenceReader));
}
}
List<ReadOnlyMemChunk> memChunks = resource.getReadOnlyMemChunk();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
index f961299..6fe01ff 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java
@@ -39,7 +39,7 @@ import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
@@ -161,7 +161,7 @@ public class UnseqTsFileRecoverTest {
TsFileSequenceReader fileReader = new TsFileSequenceReader(tsF.getPath(), true);
IMetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(fileReader);
- IChunkLoader chunkLoader = new ChunkLoaderImpl(fileReader);
+ IChunkLoader chunkLoader = new CachedChunkLoaderImpl(fileReader);
Path path = new Path("device1", "sensor1");
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
index 89b64db..c4e9fbb 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java
@@ -20,7 +20,7 @@ package org.apache.iotdb.tsfile.read;
import java.io.IOException;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
@@ -40,7 +40,7 @@ public class ReadOnlyTsFile implements AutoCloseable {
public ReadOnlyTsFile(TsFileSequenceReader fileReader) throws IOException {
this.fileReader = fileReader;
this.metadataQuerier = new MetadataQuerierByFileImpl(fileReader);
- this.chunkLoader = new ChunkLoaderImpl(fileReader);
+ this.chunkLoader = new CachedChunkLoaderImpl(fileReader);
tsFileExecutor = new TsFileExecutor(metadataQuerier, chunkLoader);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
similarity index 90%
rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
index a1935d5..fbe7d92 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java
@@ -28,13 +28,13 @@ import java.io.IOException;
/**
* Read one Chunk and cache it into a LRUCache.
*/
-public class ChunkLoaderImpl implements IChunkLoader {
+public class CachedChunkLoaderImpl implements IChunkLoader {
private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000;
private TsFileSequenceReader reader;
private LRUCache<ChunkMetaData, Chunk> chunkCache;
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
+ public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) {
this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE);
}
@@ -44,7 +44,7 @@ public class ChunkLoaderImpl implements IChunkLoader {
* @param fileSequenceReader file sequence reader
* @param cacheSize cache size
*/
- public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
+ public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) {
this.reader = fileSequenceReader;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
index 19a8416..5094bd2 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java
@@ -54,7 +54,7 @@ public class ChunkLoaderTest {
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d2.s1"));
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
for (ChunkMetaData chunkMetaData : chunkMetaDataList) {
Chunk chunk = seriesChunkLoader.getChunk(chunkMetaData);
ChunkHeader chunkHeader = chunk.getHeader();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
index 54eacc0..28d772c 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
@@ -59,7 +59,7 @@ public class QueryExecutorTest {
TsFileGeneratorForTest.generateFile(rowCount, 16 * 1024 * 1024, 10000);
fileReader = new TsFileSequenceReader(FILE_PATH);
metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader);
- chunkLoader = new ChunkLoaderImpl(fileReader);
+ chunkLoader = new CachedChunkLoaderImpl(fileReader);
queryExecutorWithQueryFilter = new TsFileExecutor(metadataQuerierByFile, chunkLoader);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
index e5818eb..3045dc8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.series.AbstractFileSeriesReader;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
@@ -61,7 +61,7 @@ public class ReaderByTimestampTest {
@Test
public void readByTimestamp() throws IOException {
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));
AbstractFileSeriesReader seriesReader = new FileSeriesReader(seriesChunkLoader,
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
index e1b9687..38bc71d 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java
@@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression;
@@ -53,7 +53,7 @@ public class TimeGeneratorTest {
TsFileGeneratorForTest.generateFile(1000, 10 * 1024 * 1024, 10000);
fileReader = new TsFileSequenceReader(FILE_PATH);
metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader);
- chunkLoader = new ChunkLoaderImpl(fileReader);
+ chunkLoader = new CachedChunkLoaderImpl(fileReader);
}
@After
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
index 93c72dc..2622ca8 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
import org.apache.iotdb.tsfile.read.filter.TimeFilter;
@@ -65,7 +65,7 @@ public class ReaderTest {
@Test
public void readTest() throws IOException {
int count = 0;
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));
@@ -101,7 +101,7 @@ public class ReaderTest {
@Test
public void readWithFilterTest() throws IOException {
- ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader);
+ CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader);
List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile
.getChunkMetaDataList(new Path("d1.s1"));
[incubator-iotdb] 02/02: Merge branch 'new_series_reader' into
CachedChunk
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch CachedChunk
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit be7abc72483959b2bb52995105d6f3988c5f6415
Merge: a74ce81 792f60c
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Fri Feb 14 11:25:25 2020 +0800
Merge branch 'new_series_reader' into CachedChunk
.../java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java | 6 +++---
.../java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java | 6 ++++++
.../java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java | 6 ++++++
.../java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java | 6 ++++++
.../java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java | 6 ++++++
.../java/org/apache/iotdb/db/utils/datastructure/IntTVList.java | 6 ++++++
.../java/org/apache/iotdb/db/utils/datastructure/LongTVList.java | 6 ++++++
.../main/java/org/apache/iotdb/db/utils/datastructure/TVList.java | 5 +++++
8 files changed, 44 insertions(+), 3 deletions(-)