You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/12/24 02:41:26 UTC
[iotdb] branch rel/0.11 updated: [To rel/0.11][IOTDB-1068] Fix Time
series metadata cache bug (#2314)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new a1ca00a [To rel/0.11][IOTDB-1068] Fix Time series metadata cache bug (#2314)
a1ca00a is described below
commit a1ca00adbc7084c532a0e30e5588117451ac1627
Author: Jackie Tien <Ja...@foxmail.com>
AuthorDate: Thu Dec 24 10:41:04 2020 +0800
[To rel/0.11][IOTDB-1068] Fix Time series metadata cache bug (#2314)
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 25 +++++++++++-----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 34 ++++++++++++++++++++++
2 files changed, 51 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 05ecabd..e49c12f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -134,6 +134,12 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
+ if (config.isDebugOn()) {
+ DEBUG_LOGGER.info(
+ "Cache miss: " + key.device + "." + key.measurement + " metadata in file: "
+ + key.filePath);
+ DEBUG_LOGGER.info("Device: " + key.device + " all sensors: " + allSensors);
+ }
// allow for the parallelism of different devices in different files
synchronized (devices
.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
@@ -148,26 +154,29 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- printCacheLog(false);
+ Path path = new Path(key.device, key.measurement);
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
- if (bloomFilter != null && !bloomFilter
- .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
-
+ if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
if (config.isDebugOn()) {
DEBUG_LOGGER.info("TimeSeries meta data " + key + " is filter by bloomFilter!");
}
return null;
}
+ printCacheLog(false);
List<TimeseriesMetadata> timeSeriesMetadataList = reader
- .readTimeseriesMetadata(key.device, allSensors);
+ .readTimeseriesMetadata(path, allSensors);
// put TimeSeriesMetadata of all sensors used in this query into cache
lock.writeLock().lock();
try {
- timeSeriesMetadataList.forEach(metadata ->
- lruCache.put(new TimeSeriesMetadataCacheKey(key.filePath, key.device,
- metadata.getMeasurementId()), metadata));
+ timeSeriesMetadataList.forEach(metadata -> {
+ TimeSeriesMetadataCacheKey k = new TimeSeriesMetadataCacheKey(key.filePath,
+ key.device, metadata.getMeasurementId());
+ if (!lruCache.containsKey(k)) {
+ lruCache.put(k, metadata);
+ }
+ });
timeseriesMetadata = lruCache.get(key);
} finally {
lock.writeLock().unlock();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 432b70b..6d79f9b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -331,6 +331,40 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
+ /**
+ * Find the leaf node that contains path, return all the sensors in that leaf node which are also
+ * in allSensors set
+ */
+ public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
+ throws IOException {
+ readFileMetadata();
+ MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
+ ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
+ if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
+ metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
+ }
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ while (buffer.hasRemaining()) {
+ TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
+ }
+ return timeseriesMetadataList;
+ }
+
public List<TimeseriesMetadata> readTimeseriesMetadata(String device, Set<String> measurements)
throws IOException {
readFileMetadata();