You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/11/02 14:33:02 UTC
[iotdb] 01/01: [IOTDB-975] Fix reading file error in
getChunkMetadataList
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_975
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ce174520be127ee9954b018895c9c259a4dd650e
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Mon Nov 2 22:31:45 2020 +0800
[IOTDB-975] Fix reading file error in getChunkMetadataList
---
.../tsfile/file/metadata/MetadataIndexNode.java | 20 ++++++++++++----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 27 +++++++++++++++-------
.../file/metadata/MetadataIndexNodeTest.java | 8 +++----
3 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
index 603aeda..95f67f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
@@ -103,12 +103,16 @@ public class MetadataIndexNode {
children.add(MetadataIndexEntry.deserializeFrom(buffer));
}
long offset = ReadWriteIOUtils.readLong(buffer);
- MetadataIndexNodeType nodeType = MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer));
+ MetadataIndexNodeType nodeType = MetadataIndexNodeType
+ .deserialize(ReadWriteIOUtils.readByte(buffer));
return new MetadataIndexNode(children, offset, nodeType);
}
- public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key) {
- int index = binarySearchInChildren(key);
+ public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key, boolean exactSearch) {
+ int index = binarySearchInChildren(key, exactSearch);
+ if (index == -1) {
+ return null;
+ }
long childEndOffset;
if (index != children.size() - 1) {
childEndOffset = children.get(index + 1).getOffset();
@@ -118,7 +122,7 @@ public class MetadataIndexNode {
return new Pair<>(children.get(index), childEndOffset);
}
- int binarySearchInChildren(String key) {
+ int binarySearchInChildren(String key, boolean exactSearch) {
int low = 0;
int high = children.size() - 1;
@@ -135,6 +139,12 @@ public class MetadataIndexNode {
return mid; // key found
}
}
- return low == 0 ? low : low - 1; // key not found
+
+ // key not found
+ if (exactSearch) {
+ return -1;
+ } else {
+ return low == 0 ? low : low - 1;
+ }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index f8a6133..2341136 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -304,13 +304,19 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
@@ -328,7 +334,7 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
int maxDegreeOfIndexNode = config.getMaxDegreeOfIndexNode();
if (measurements.size() > maxDegreeOfIndexNode / Math.log(maxDegreeOfIndexNode)) {
@@ -344,7 +350,7 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
@@ -568,7 +574,7 @@ public class TsFileSequenceReader implements AutoCloseable {
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
generateMetadataIndex(metadataIndexPair.left, buffer, device,
@@ -594,13 +600,18 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return target MetadataIndexEntry, endOffset pair
*/
private Pair<MetadataIndexEntry, Long> getMetadataAndEndOffset(MetadataIndexNode metadataIndex,
- String name, MetadataIndexNodeType type) throws IOException {
- Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex.getChildIndexEntry(name);
+ String name, MetadataIndexNodeType type, boolean exactSearch) throws IOException {
+ Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex
+ .getChildIndexEntry(name, exactSearch);
+ if (childIndexEntry == null) {
+ return null;
+ }
if (!metadataIndex.getNodeType().equals(type)) {
return childIndexEntry;
}
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
- return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type);
+ return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type,
+ exactSearch);
}
/**
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
index eb1ef3a..72a84d7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
@@ -38,9 +38,9 @@ public class MetadataIndexNodeTest {
MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L,
MetadataIndexNodeType.LEAF_MEASUREMENT);
- Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13"));
- Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23"));
+ Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false));
+ Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false));
}
}