You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/01/04 12:28:15 UTC
[iotdb] branch master updated: [IOTDB-1068] Fix Time series
metadata cache bug (#2316)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 85d4a00 [IOTDB-1068] Fix Time series metadata cache bug (#2316)
85d4a00 is described below
commit 85d4a00305c283f28440e8bf3705dbff168c090c
Author: Jackie Tien <Ja...@foxmail.com>
AuthorDate: Mon Jan 4 20:27:51 2021 +0800
[IOTDB-1068] Fix Time series metadata cache bug (#2316)
[IOTDB-1068] Fix Time series metadata cache bug
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 32 +++++---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 34 +++++++++
.../tsfile/read/TimeSeriesMetadataReadTest.java | 87 ++++++++++++++++++++++
.../apache/iotdb/tsfile/utils/FileGenerator.java | 22 +++++-
4 files changed, 161 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 474ba38..384a496 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -133,6 +133,11 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
+ if (config.isDebugOn()) {
+ DEBUG_LOGGER
+ .info("Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
+ DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
+ }
// allow for the parallelism of different devices
synchronized (devices
.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
@@ -147,26 +152,29 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- printCacheLog(false);
+ Path path = new Path(key.device, key.measurement);
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
- if (bloomFilter != null && !bloomFilter
- .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
-
+ if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
if (config.isDebugOn()) {
- DEBUG_LOGGER.info("TimeSeries meta data " + key + " is filter by bloomFilter!");
+ DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
return null;
}
+ printCacheLog(false);
List<TimeseriesMetadata> timeSeriesMetadataList = reader
- .readTimeseriesMetadata(key.device, allSensors);
+ .readTimeseriesMetadata(path, allSensors);
// put TimeSeriesMetadata of all sensors used in this query into cache
lock.writeLock().lock();
try {
- timeSeriesMetadataList.forEach(metadata ->
- lruCache.put(new TimeSeriesMetadataCacheKey(key.filePath, key.device,
- metadata.getMeasurementId()), metadata));
+ timeSeriesMetadataList.forEach(metadata -> {
+ TimeSeriesMetadataCacheKey k = new TimeSeriesMetadataCacheKey(key.filePath,
+ key.device, metadata.getMeasurementId());
+ if (!lruCache.containsKey(k)) {
+ lruCache.put(k, metadata);
+ }
+ });
timeseriesMetadata = lruCache.get(key);
} finally {
lock.writeLock().unlock();
@@ -176,14 +184,14 @@ public class TimeSeriesMetadataCache {
}
if (timeseriesMetadata == null) {
if (config.isDebugOn()) {
- DEBUG_LOGGER.info("The file doesn't have this time series " + key);
+ DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
}
return null;
} else {
if (config.isDebugOn()) {
DEBUG_LOGGER.info(
- "Get timeseries: " + key.device + "." + key.measurement + " metadata in file: "
- + key.filePath + " from cache: " + timeseriesMetadata);
+ "Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device,
+ key.measurement, key.filePath, timeseriesMetadata);
}
return new TimeseriesMetadata(timeseriesMetadata);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 6bdaaf2..2abb60c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -333,6 +333,40 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
+ /**
+ * Find the leaf node that contains path, return all the sensors in that leaf node which are also
+ * in allSensors set
+ */
+ public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
+ throws IOException {
+ readFileMetadata();
+ MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
+ ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
+ if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
+ metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
+ }
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ while (buffer.hasRemaining()) {
+ TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
+ }
+ return timeseriesMetadataList;
+ }
+
public List<TimeseriesMetadata> readTimeseriesMetadata(String device, Set<String> measurements)
throws IOException {
readFileMetadata();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java
new file mode 100644
index 0000000..006ce51
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.read;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.constant.TestConstant;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.FileGenerator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TimeSeriesMetadataReadTest {
+
+ private static final String FILE_PATH = TestConstant.BASE_OUTPUT_PATH
+ .concat("TimeSeriesMetadataReadTest.tsfile");
+ private final TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+ private int maxDegreeOfIndexNode;
+
+ @Before
+ public void before() throws IOException {
+ int rowCount = 100;
+ maxDegreeOfIndexNode = conf.getMaxDegreeOfIndexNode();
+ conf.setMaxDegreeOfIndexNode(3);
+ FileGenerator.generateFile(rowCount, 10000, FILE_PATH);
+ }
+
+ @After
+ public void after() throws IOException {
+ FileGenerator.after();
+ conf.setMaxDegreeOfIndexNode(maxDegreeOfIndexNode);
+ }
+
+ @Test
+ public void testReadTimeseriesMetadata() throws IOException {
+ TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH);
+ Path path = new Path("d1", "s1");
+ Set<String> set = new HashSet<>();
+ set.add("s1");
+ set.add("s2");
+ set.add("s3");
+ // the Max Degree Of Index Node is set to be 3, so the leaf node should only contains 3 sensors
+ // s4 should not be returned as result
+ set.add("s4");
+ List<TimeseriesMetadata> timeseriesMetadataList = reader.readTimeseriesMetadata(path, set);
+ Assert.assertEquals(3, timeseriesMetadataList.size());
+ for (int i = 1; i <= timeseriesMetadataList.size(); i++) {
+ Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 1).getMeasurementId());
+ }
+
+ path = new Path("d1", "s5");
+ set.clear();
+ set.add("s5");
+ set.add("s6");
+ // this is a fake one, this file doesn't contain this measurement
+ // so the result is not supposed to contain this measurement's timeseries metadata
+ set.add("s8");
+ timeseriesMetadataList = reader.readTimeseriesMetadata(path, set);
+ Assert.assertEquals(2, timeseriesMetadataList.size());
+ for (int i = 5; i < 7; i++) {
+ Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 5).getMeasurementId());
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
index e2e1b56..561ae03 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
@@ -60,6 +60,16 @@ public class FileGenerator {
config.setMaxNumberOfPointsInPage(oldMaxNumberOfPointsInPage);
}
+ public static void generateFile(int rowCount, int maxNumberOfPointsInPage, String filePath) throws IOException {
+ ROW_COUNT = rowCount;
+ int oldMaxNumberOfPointsInPage = config.getMaxNumberOfPointsInPage();
+ config.setMaxNumberOfPointsInPage(maxNumberOfPointsInPage);
+
+ prepare();
+ write(filePath);
+ config.setMaxNumberOfPointsInPage(oldMaxNumberOfPointsInPage);
+ }
+
public static void generateFile(int maxNumberOfPointsInPage, int deviceNum,
int measurementNum) throws IOException {
ROW_COUNT = 1;
@@ -90,11 +100,15 @@ public class FileGenerator {
}
public static void after() {
+ after(outputDataFile);
+ }
+
+ public static void after(String filePath) {
File file = new File(inputDataFile);
if (file.exists()) {
file.delete();
}
- file = new File(outputDataFile);
+ file = new File(filePath);
if (file.exists()) {
file.delete();
}
@@ -184,7 +198,11 @@ public class FileGenerator {
}
static public void write() throws IOException {
- File file = new File(outputDataFile);
+ write(outputDataFile);
+ }
+
+ static public void write(String filePath) throws IOException {
+ File file = new File(filePath);
File errorFile = new File(errorOutputDataFile);
if (file.exists()) {
file.delete();