You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/05/06 13:56:41 UTC
[incubator-iotdb] branch jira641 created (now ffa0084)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch jira641
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at ffa0084 Remove TsFileMetadataCache
This branch includes the following new commits:
new ffa0084 Remove TsFileMetadataCache
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: Remove TsFileMetadataCache
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch jira641
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit ffa0084dc783e2b5d7a33bcf0cddbc8afb781919
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed May 6 21:56:07 2020 +0800
Remove TsFileMetadataCache
---
.../db/engine/cache/CacheHitRatioMonitor.java | 6 -
.../engine/cache/CacheHitRatioMonitorMXBean.java | 2 -
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 10 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 11 +-
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 174 ---------------------
.../iotdb/db/engine/merge/task/MergeFileTask.java | 3 -
.../apache/iotdb/db/engine/merge/MergeTest.java | 2 -
.../query/reader/series/SeriesReaderTestUtil.java | 3 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 3 -
.../iotdb/tsfile/read/TsFileSequenceReader.java | 11 ++
10 files changed, 19 insertions(+), 206 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
index 457cc8a..3d5f4f5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java
@@ -61,12 +61,6 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic
}
@Override
- public double getTsfileMetaDataHitRatio() {
- tsfileMetaDataHitRatio = TsFileMetaDataCache.getInstance().calculateTsFileMetaDataHitRatio();
- return tsfileMetaDataHitRatio;
- }
-
- @Override
public double getChunkHitRatio() {
return ChunkCache.getInstance().calculateChunkHitRatio();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
index f546ef0..644374d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java
@@ -22,8 +22,6 @@ public interface CacheHitRatioMonitorMXBean {
double getChunkMetaDataHitRatio();
- double getTsfileMetaDataHitRatio();
-
double getChunkHitRatio();
double getTimeSeriesMetadataHitRatio();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 3ed15cb..2e8f294 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -32,7 +32,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
@@ -102,8 +101,8 @@ public class ChunkMetadataCache {
throws IOException {
if (!cacheEnable) {
// bloom filter part
- TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
- BloomFilter bloomFilter = fileMetaData.getBloomFilter();
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ BloomFilter bloomFilter = tsFileReader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) {
if (logger.isDebugEnabled()) {
logger.debug(String
@@ -112,7 +111,6 @@ public class ChunkMetadataCache {
return new ArrayList<>();
}
// If timeseries isn't included in the tsfile, empty list is returned.
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
return tsFileReader.getChunkMetadataList(seriesPath);
}
@@ -141,8 +139,8 @@ public class ChunkMetadataCache {
}
printCacheLog(false);
// bloom filter part
- TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
- BloomFilter bloomFilter = fileMetaData.getBloomFilter();
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ BloomFilter bloomFilter = tsFileReader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) {
return new ArrayList<>();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index e620cd1..ccd430f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -31,7 +31,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
@@ -91,13 +90,12 @@ public class TimeSeriesMetadataCache {
throws IOException {
if (!cacheEnable) {
// bloom filter part
- TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath);
- BloomFilter bloomFilter = fileMetaData.getBloomFilter();
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter
.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
return reader.readTimeseriesMetadata(new Path(key.device, key.measurement));
}
@@ -123,13 +121,12 @@ public class TimeSeriesMetadataCache {
}
printCacheLog(false);
// bloom filter part
- TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath);
- BloomFilter bloomFilter = fileMetaData.getBloomFilter();
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter
.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
List<TimeseriesMetadata> timeSeriesMetadataList = reader
.readTimeseriesMetadata(key.device, allSensors);
// put TimeSeriesMetadata of all sensors used in this query into cache
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
deleted file mode 100644
index ec59e95..0000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.utils.FileLoaderUtils;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is used to cache <code>TsFileMetaData</code> of tsfile in IoTDB.
- */
-public class TsFileMetaDataCache {
-
- private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class);
- private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
- private static boolean cacheEnable = config.isMetaDataCacheEnable();
- private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForFileMetaDataCache();
-
- /**
- * TsFile path -> TsFileMetaData
- */
- private final LRULinkedHashMap<String, TsFileMetadata> cache;
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private AtomicLong cacheHitNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
-
- /**
- * estimated size of metadataIndex entry in TsFileMetaData.
- */
- private long metadataIndexEntrySize = 0;
-
- private TsFileMetaDataCache() {
- logger.info("TsFileMetaDataCache size = " + MEMORY_THRESHOLD_IN_B);
- cache = new LRULinkedHashMap<String, TsFileMetadata>(MEMORY_THRESHOLD_IN_B, true) {
- @Override
- protected long calEntrySize(String key, TsFileMetadata value) {
- if (metadataIndexEntrySize == 0 && value.getMetadataIndex() != null
- && !value.getMetadataIndex().getChildren().isEmpty()) {
- metadataIndexEntrySize = RamUsageEstimator
- .sizeOf(value.getMetadataIndex().getChildren().iterator().next());
- }
- // totalChunkNum, invalidChunkNum
- long valueSize = 4 + 4L;
-
- // metadataIndex
- if (value.getMetadataIndex() != null) {
- valueSize += value.getMetadataIndex().getChildren().size() * metadataIndexEntrySize;
- }
-
- // versionInfo
- if (value.getVersionInfo() != null) {
- valueSize += value.getVersionInfo().size() * 16;
- }
- return key.getBytes(TSFileConfig.STRING_CHARSET).length + valueSize;
- }
- };
- }
-
- public static TsFileMetaDataCache getInstance() {
- return TsFileMetaDataCacheHolder.INSTANCE;
- }
-
- /**
- * get the TsFileMetaData for given TsFile.
- *
- * @param filePath -given TsFile
- */
- public TsFileMetadata get(String filePath) throws IOException {
- if (!cacheEnable) {
- return FileLoaderUtils.getTsFileMetadata(filePath);
- }
-
- cacheRequestNum.incrementAndGet();
-
- lock.readLock().lock();
- try {
- if (cache.containsKey(filePath)) {
- cacheHitNum.incrementAndGet();
- printCacheLog(true);
- return cache.get(filePath);
- }
- } finally {
- lock.readLock().unlock();
- }
-
- lock.writeLock().lock();
- try {
- if (cache.containsKey(filePath)) {
- cacheHitNum.incrementAndGet();
- printCacheLog(true);
- return cache.get(filePath);
- }
- printCacheLog(false);
- TsFileMetadata fileMetaData = FileLoaderUtils.getTsFileMetadata(filePath);
- cache.put(filePath, fileMetaData);
- return fileMetaData;
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- private void printCacheLog(boolean isHit) {
- if (!logger.isDebugEnabled()) {
- return;
- }
- if (isHit) {
- logger.debug(
- "[TsFileMetaData cache hit] The number of requests for cache is {}, hit rate is {}.",
- cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
- } else {
- logger.debug(
- "[TsFileMetaData cache didn't hit] The number of requests for cache is {}, hit rate is {}.",
- cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
- }
- }
-
- double calculateTsFileMetaDataHitRatio() {
- if (cacheRequestNum.get() != 0) {
- return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
- } else {
- return 0;
- }
- }
-
- public void remove(TsFileResource resource) {
- synchronized (cache) {
- cache.remove(resource.getPath());
- }
- }
-
- public void clear() {
- synchronized (cache) {
- cache.clear();
- }
- }
-
- /**
- * Singleton pattern
- */
- private static class TsFileMetaDataCacheHolder {
-
- private TsFileMetaDataCacheHolder() {
- }
-
- private static final TsFileMetaDataCache INSTANCE = new TsFileMetaDataCache();
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index b6a5778..006ec5c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -30,7 +30,6 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
-import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeContext;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
@@ -124,7 +123,6 @@ class MergeFileTask {
seqFile.getWriteQueryLock().writeLock().lock();
try {
- TsFileMetaDataCache.getInstance().remove(seqFile);
ChunkMetadataCache.getInstance().remove(seqFile);
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getPath());
@@ -262,7 +260,6 @@ class MergeFileTask {
seqFile.getWriteQueryLock().writeLock().lock();
try {
resource.removeFileReader(seqFile);
- TsFileMetaDataCache.getInstance().remove(seqFile);
ChunkMetadataCache.getInstance().remove(seqFile);
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getPath());
seqFile.getFile().delete();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index f626830..55cee97 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
-import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -87,7 +86,6 @@ abstract class MergeTest {
seqResources.clear();
unseqResources.clear();
IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold);
- TsFileMetaDataCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
MManager.getInstance().clear();
EnvironmentUtils.cleanAllDir();
diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
index 74689eb..58177fd 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
@@ -31,7 +31,6 @@ import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
-import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -43,7 +42,6 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -73,7 +71,6 @@ public class SeriesReaderTestUtil {
removeFiles(seqResources, unseqResources);
seqResources.clear();
unseqResources.clear();
- TsFileMetaDataCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
MManager.getInstance().clear();
EnvironmentUtils.cleanAllDir();
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 94b5e73..d64a8b3 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -31,14 +31,12 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.authorizer.LocalFileAuthorizer;
import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
-import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -134,7 +132,6 @@ public class EnvironmentUtils {
// clean cache
if (config.isMetaDataCacheEnable()) {
- TsFileMetaDataCache.getInstance().clear();
ChunkMetadataCache.getInstance().clear();
}
// close metadata
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index efb5c31..0bd8e92 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -56,6 +56,7 @@ import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
+import org.apache.iotdb.tsfile.utils.BloomFilter;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.utils.VersionUtils;
@@ -263,6 +264,16 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
+ * this function does not modify the position of the file reader.
+ *
+ * @throws IOException io error
+ */
+ public BloomFilter readBloomFilter() throws IOException {
+ readFileMetadata();
+ return tsFileMetaData.getBloomFilter();
+ }
+
+ /**
* this function reads measurements and TimeseriesMetaDatas in given device Thread Safe
*
* @param device name