You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/10/27 10:38:37 UTC
[iotdb] 01/01: improve chunk metadata cache
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch CacheBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 49cba854b2778a749c2c31922c1f0a1622b70e09
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Tue Oct 27 18:37:13 2020 +0800
improve chunk metadata cache
---
.../apache/iotdb/db/engine/cache/ChunkMetadataCache.java | 15 +++++----------
.../reader/chunk/metadata/DiskChunkMetadataLoader.java | 9 +++++----
.../reader/chunk/metadata/MemChunkMetadataLoader.java | 3 ++-
.../iotdb/tsfile/file/metadata/TimeseriesMetadata.java | 2 +-
.../tsfile/read/controller/IChunkMetadataLoader.java | 3 ++-
5 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 4ff58e7..1d511d7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
@@ -106,8 +107,8 @@ public class ChunkMetadataCache {
/**
* get {@link ChunkMetadata}. THREAD SAFE.
*/
- public List<ChunkMetadata> get(String filePath, Path seriesPath)
- throws IOException {
+ public List<ChunkMetadata> get(String filePath, Path seriesPath,
+ TimeseriesMetadata timeSeriesMetadata) throws IOException {
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
@@ -121,7 +122,7 @@ public class ChunkMetadataCache {
return new ArrayList<>();
}
// If timeseries isn't included in the tsfile, empty list is returned.
- return tsFileReader.getChunkMetadataList(seriesPath);
+ return tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
}
AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR
@@ -137,19 +138,13 @@ public class ChunkMetadataCache {
lock.readLock().unlock();
}
-
if (chunkMetadataList != null) {
printCacheLog(true);
cacheHitNum.incrementAndGet();
} else {
printCacheLog(false);
- // bloom filter part
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
- BloomFilter bloomFilter = tsFileReader.readBloomFilter();
- if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) {
- return new ArrayList<>();
- }
- chunkMetadataList = FileLoaderUtils.getChunkMetadataList(seriesPath, filePath);
+ chunkMetadataList = tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
lock.writeLock().lock();
try {
lruCache.put(key, chunkMetadataList);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 877647b..6b2bfa9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.query.reader.chunk.metadata;
+import java.io.IOException;
+import java.util.List;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -25,11 +27,10 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import java.io.IOException;
-import java.util.List;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@@ -47,9 +48,9 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
+ public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException {
List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
- .getInstance().get(resource.getPath(), seriesPath);
+ .getInstance().get(resource.getPath(), seriesPath, timeseriesMetadata);
setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 8ab268c..aa1e7cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -44,7 +45,7 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList() {
+ public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) {
List<ChunkMetadata> chunkMetadataList = resource.getChunkMetadataList();
DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 4893563..60dce34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -130,7 +130,7 @@ public class TimeseriesMetadata implements Accountable {
}
public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
- return chunkMetadataLoader.loadChunkMetadataList();
+ return chunkMetadataLoader.loadChunkMetadataList(this);
}
public boolean isModified() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index f80c42f..9badcda 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -22,13 +22,14 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
public interface IChunkMetadataLoader {
/**
* read all chunk metadata of one time series in one file.
*/
- List<ChunkMetadata> loadChunkMetadataList() throws IOException;
+ List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException;
/**
* For query 0.9/v1 tsfile only