You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/11/02 14:58:57 UTC
[iotdb] branch jira_975_master created (now 748def3)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch jira_975_master
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 748def3 [IOTDB-975] Fix reading file error in getChunkMetadataList
This branch includes the following new commits:
new 748def3 [IOTDB-975] Fix reading file error in getChunkMetadataList
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-975] Fix reading file error in
getChunkMetadataList
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_975_master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 748def3e5ec1644dd6d2ce76751ebe316b607d59
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Mon Nov 2 22:58:20 2020 +0800
[IOTDB-975] Fix reading file error in getChunkMetadataList
---
.../tsfile/file/metadata/MetadataIndexNode.java | 20 ++++++++++++----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 27 +++++++++++++++-------
.../file/metadata/MetadataIndexNodeTest.java | 8 +++----
3 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
index 603aeda..95f67f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
@@ -103,12 +103,16 @@ public class MetadataIndexNode {
children.add(MetadataIndexEntry.deserializeFrom(buffer));
}
long offset = ReadWriteIOUtils.readLong(buffer);
- MetadataIndexNodeType nodeType = MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer));
+ MetadataIndexNodeType nodeType = MetadataIndexNodeType
+ .deserialize(ReadWriteIOUtils.readByte(buffer));
return new MetadataIndexNode(children, offset, nodeType);
}
- public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key) {
- int index = binarySearchInChildren(key);
+ public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key, boolean exactSearch) {
+ int index = binarySearchInChildren(key, exactSearch);
+ if (index == -1) {
+ return null;
+ }
long childEndOffset;
if (index != children.size() - 1) {
childEndOffset = children.get(index + 1).getOffset();
@@ -118,7 +122,7 @@ public class MetadataIndexNode {
return new Pair<>(children.get(index), childEndOffset);
}
- int binarySearchInChildren(String key) {
+ int binarySearchInChildren(String key, boolean exactSearch) {
int low = 0;
int high = children.size() - 1;
@@ -135,6 +139,12 @@ public class MetadataIndexNode {
return mid; // key found
}
}
- return low == 0 ? low : low - 1; // key not found
+
+ // key not found
+ if (exactSearch) {
+ return -1;
+ } else {
+ return low == 0 ? low : low - 1;
+ }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 0fa4d5a..f49a7dc 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -305,13 +305,19 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
@@ -329,7 +335,7 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
List<String> measurementList = new ArrayList<>(measurements);
Set<String> measurementsHadFound = new HashSet<>();
@@ -344,7 +350,7 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
@@ -588,7 +594,7 @@ public class TsFileSequenceReader implements AutoCloseable {
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
generateMetadataIndex(metadataIndexPair.left, buffer, device,
@@ -614,13 +620,18 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return target MetadataIndexEntry, endOffset pair
*/
private Pair<MetadataIndexEntry, Long> getMetadataAndEndOffset(MetadataIndexNode metadataIndex,
- String name, MetadataIndexNodeType type) throws IOException {
- Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex.getChildIndexEntry(name);
+ String name, MetadataIndexNodeType type, boolean exactSearch) throws IOException {
+ Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex
+ .getChildIndexEntry(name, exactSearch);
+ if (childIndexEntry == null) {
+ return null;
+ }
if (!metadataIndex.getNodeType().equals(type)) {
return childIndexEntry;
}
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
- return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type);
+ return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type,
+ exactSearch);
}
/**
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
index eb1ef3a..72a84d7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
@@ -38,9 +38,9 @@ public class MetadataIndexNodeTest {
MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L,
MetadataIndexNodeType.LEAF_MEASUREMENT);
- Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13"));
- Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23"));
+ Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false));
+ Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false));
}
}