You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/11/03 06:56:02 UTC
[iotdb] branch rel/0.10 updated: [IOTDB-975] [To rel/0.10] Fix
reading file error in getChunkMetadataList (#1919)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.10 by this push:
new 984d3bb [IOTDB-975] [To rel/0.10] Fix reading file error in getChunkMetadataList (#1919)
984d3bb is described below
commit 984d3bb7bb0de4c570115d262553a9386290cf32
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Nov 3 14:55:01 2020 +0800
[IOTDB-975] [To rel/0.10] Fix reading file error in getChunkMetadataList (#1919)
---
.../tsfile/file/metadata/MetadataIndexNode.java | 20 ++++++++---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 42 +++++++++++++++++-----
.../file/metadata/MetadataIndexNodeTest.java | 8 ++---
3 files changed, 52 insertions(+), 18 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
index 603aeda..95f67f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
@@ -103,12 +103,16 @@ public class MetadataIndexNode {
children.add(MetadataIndexEntry.deserializeFrom(buffer));
}
long offset = ReadWriteIOUtils.readLong(buffer);
- MetadataIndexNodeType nodeType = MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer));
+ MetadataIndexNodeType nodeType = MetadataIndexNodeType
+ .deserialize(ReadWriteIOUtils.readByte(buffer));
return new MetadataIndexNode(children, offset, nodeType);
}
- public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key) {
- int index = binarySearchInChildren(key);
+ public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key, boolean exactSearch) {
+ int index = binarySearchInChildren(key, exactSearch);
+ if (index == -1) {
+ return null;
+ }
long childEndOffset;
if (index != children.size() - 1) {
childEndOffset = children.get(index + 1).getOffset();
@@ -118,7 +122,7 @@ public class MetadataIndexNode {
return new Pair<>(children.get(index), childEndOffset);
}
- int binarySearchInChildren(String key) {
+ int binarySearchInChildren(String key, boolean exactSearch) {
int low = 0;
int high = children.size() - 1;
@@ -135,6 +139,12 @@ public class MetadataIndexNode {
return mid; // key found
}
}
- return low == 0 ? low : low - 1; // key not found
+
+ // key not found
+ if (exactSearch) {
+ return -1;
+ } else {
+ return low == 0 ? low : low - 1;
+ }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index f8a6133..0affa42 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -304,13 +305,19 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
@@ -328,7 +335,10 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
+ if (metadataIndexPair == null) {
+ return Collections.emptyList();
+ }
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
int maxDegreeOfIndexNode = config.getMaxDegreeOfIndexNode();
if (measurements.size() > maxDegreeOfIndexNode / Math.log(maxDegreeOfIndexNode)) {
@@ -344,7 +354,10 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (measurementMetadataIndexPair == null) {
+ return Collections.emptyList();
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
@@ -568,7 +581,10 @@ public class TsFileSequenceReader implements AutoCloseable {
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
+ if (metadataIndexPair == null) {
+ return Collections.emptyList();
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
generateMetadataIndex(metadataIndexPair.left, buffer, device,
@@ -591,16 +607,24 @@ public class TsFileSequenceReader implements AutoCloseable {
* return when it is not INTERNAL_MEASUREMENT. This works for the situation
* when the index tree does NOT have the device level and ONLY has the
* measurement level.
+ * @param exactSearch if is in exact search mode, return null when there is no entry with name;
+ * or else return the nearest MetadataIndexEntry before it (for deeper
+ * search)
* @return target MetadataIndexEntry, endOffset pair
*/
private Pair<MetadataIndexEntry, Long> getMetadataAndEndOffset(MetadataIndexNode metadataIndex,
- String name, MetadataIndexNodeType type) throws IOException {
- Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex.getChildIndexEntry(name);
+ String name, MetadataIndexNodeType type, boolean exactSearch) throws IOException {
+ Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex
+ .getChildIndexEntry(name, exactSearch);
+ if (childIndexEntry == null) {
+ return null;
+ }
if (!metadataIndex.getNodeType().equals(type)) {
return childIndexEntry;
}
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
- return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type);
+ return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type,
+ exactSearch);
}
/**
@@ -942,7 +966,7 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadata(path);
if (timeseriesMetaData == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
}
List<ChunkMetadata> chunkMetadataList = readChunkMetaDataList(timeseriesMetaData);
chunkMetadataList.sort(Comparator.comparingLong(ChunkMetadata::getStartTime));
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
index eb1ef3a..72a84d7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
@@ -38,9 +38,9 @@ public class MetadataIndexNodeTest {
MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L,
MetadataIndexNodeType.LEAF_MEASUREMENT);
- Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13"));
- Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23"));
+ Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false));
+ Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false));
}
}