You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/11/01 03:03:54 UTC
[iotdb] branch rel/0.10 updated: [To rel/0.10] [IOTDB-962] Improve
the chunk metadata cache process (#1873)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.10 by this push:
new b96bf61 [To rel/0.10] [IOTDB-962] Improve the chunk metadata cache process (#1873)
b96bf61 is described below
commit b96bf61f49613f7ac9730ff5c1cf70fc27f57585
Author: Jackie Tien <Ja...@foxmail.com>
AuthorDate: Sun Nov 1 11:03:41 2020 +0800
[To rel/0.10] [IOTDB-962] Improve the chunk metadata cache process (#1873)
---
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 30 +++++++---------------
.../iotdb/db/engine/merge/task/MergeFileTask.java | 9 +++++++
.../engine/storagegroup/StorageGroupProcessor.java | 9 +++++++
.../chunk/metadata/DiskChunkMetadataLoader.java | 9 ++++---
.../chunk/metadata/MemChunkMetadataLoader.java | 3 ++-
.../db/engine/cache/ChunkMetadataCacheTest.java | 4 +--
.../apache/iotdb/db/engine/merge/MergeTest.java | 4 +++
.../query/reader/series/SeriesReaderTestUtil.java | 4 +++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 4 +++
.../tsfile/file/metadata/TimeseriesMetadata.java | 2 +-
.../read/controller/IChunkMetadataLoader.java | 3 ++-
11 files changed, 51 insertions(+), 30 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 4ff58e7..5ca9010 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
@@ -30,12 +31,11 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,22 +106,16 @@ public class ChunkMetadataCache {
/**
* get {@link ChunkMetadata}. THREAD SAFE.
*/
- public List<ChunkMetadata> get(String filePath, Path seriesPath)
- throws IOException {
+ public List<ChunkMetadata> get(String filePath, Path seriesPath,
+ TimeseriesMetadata timeSeriesMetadata) throws IOException {
+ if (timeSeriesMetadata == null) {
+ return Collections.emptyList();
+ }
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
- BloomFilter bloomFilter = tsFileReader.readBloomFilter();
- if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) {
- if (logger.isDebugEnabled()) {
- logger.debug(String
- .format("path not found by bloom filter, file is: %s, path is: %s", filePath,
- seriesPath));
- }
- return new ArrayList<>();
- }
// If timeseries isn't included in the tsfile, empty list is returned.
- return tsFileReader.getChunkMetadataList(seriesPath);
+ return tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
}
AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR
@@ -137,19 +131,13 @@ public class ChunkMetadataCache {
lock.readLock().unlock();
}
-
if (chunkMetadataList != null) {
printCacheLog(true);
cacheHitNum.incrementAndGet();
} else {
printCacheLog(false);
- // bloom filter part
TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
- BloomFilter bloomFilter = tsFileReader.readBloomFilter();
- if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) {
- return new ArrayList<>();
- }
- chunkMetadataList = FileLoaderUtils.getChunkMetadataList(seriesPath, filePath);
+ chunkMetadataList = tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
lock.writeLock().lock();
try {
lruCache.put(key, chunkMetadataList);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index 006ec5c..378e394 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -29,7 +29,10 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeContext;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
@@ -274,6 +277,12 @@ class MergeFileTask {
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
+ // clean cache
+ if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
+ ChunkCache.getInstance().clear();
+ ChunkMetadataCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ }
seqFile.getWriteQueryLock().writeLock().unlock();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 70c9064..15213dd 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -45,6 +45,9 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
@@ -1612,6 +1615,12 @@ public class StorageGroupProcessor {
try {
unseqFile.remove();
} finally {
+ // clean cache
+ if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
+ ChunkCache.getInstance().clear();
+ ChunkMetadataCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
+ }
unseqFile.getWriteQueryLock().writeLock().unlock();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 877647b..6b2bfa9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.query.reader.chunk.metadata;
+import java.io.IOException;
+import java.util.List;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -25,11 +27,10 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import java.io.IOException;
-import java.util.List;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@@ -47,9 +48,9 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
+ public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException {
List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
- .getInstance().get(resource.getPath(), seriesPath);
+ .getInstance().get(resource.getPath(), seriesPath, timeseriesMetadata);
setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 8ab268c..aa1e7cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -44,7 +45,7 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList() {
+ public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) {
List<ChunkMetadata> chunkMetadataList = resource.getChunkMetadataList();
DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
index 2926973..0766fed 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
@@ -135,7 +135,7 @@ public class ChunkMetadataCacheTest {
Assert.assertFalse(unseqResources.get(3).isClosed());
List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
- .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5));
+ .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5), null);
Assert.assertEquals(0, metaDataList.size());
}
@@ -157,7 +157,7 @@ public class ChunkMetadataCacheTest {
Assert.assertFalse(unseqResources.get(3).isClosed());
List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
- .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5));
+ .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5), null);
Assert.assertEquals(0, metaDataList.size());
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 55cee97..691c02f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -29,7 +29,9 @@ import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -86,7 +88,9 @@ abstract class MergeTest {
seqResources.clear();
unseqResources.clear();
IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold);
+ ChunkCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
MManager.getInstance().clear();
EnvironmentUtils.cleanAllDir();
MergeManager.getINSTANCE().stop();
diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
index 58177fd..5466366 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
@@ -30,7 +30,9 @@ import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -71,7 +73,9 @@ public class SeriesReaderTestUtil {
removeFiles(seqResources, unseqResources);
seqResources.clear();
unseqResources.clear();
+ ChunkCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
MManager.getInstance().clear();
EnvironmentUtils.cleanAllDir();
MergeManager.getINSTANCE().stop();
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 6e1ff67..1e06653 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -36,7 +36,9 @@ import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -132,7 +134,9 @@ public class EnvironmentUtils {
// clean cache
if (config.isMetaDataCacheEnable()) {
+ ChunkCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
+ TimeSeriesMetadataCache.getInstance().clear();
}
// close metadata
MManager.getInstance().clear();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 4893563..60dce34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -130,7 +130,7 @@ public class TimeseriesMetadata implements Accountable {
}
public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
- return chunkMetadataLoader.loadChunkMetadataList();
+ return chunkMetadataLoader.loadChunkMetadataList(this);
}
public boolean isModified() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index f80c42f..9badcda 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -22,13 +22,14 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
public interface IChunkMetadataLoader {
/**
* read all chunk metadata of one time series in one file.
*/
- List<ChunkMetadata> loadChunkMetadataList() throws IOException;
+ List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException;
/**
* For query 0.9/v1 tsfile only