You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/11/03 07:00:47 UTC
[iotdb] branch master updated: [IOTDB-975] Fix reading file error
in getChunkMetadataList (#1920)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9f09a93 [IOTDB-975] Fix reading file error in getChunkMetadataList (#1920)
9f09a93 is described below
commit 9f09a939c0ca5cc741fed1f37d4907e3910dd38c
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Tue Nov 3 15:00:34 2020 +0800
[IOTDB-975] Fix reading file error in getChunkMetadataList (#1920)
---
.../tsfile/file/metadata/MetadataIndexNode.java | 20 ++++++++---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 42 +++++++++++++++++-----
.../file/metadata/MetadataIndexNodeTest.java | 10 +++---
3 files changed, 54 insertions(+), 18 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
index 603aeda..95f67f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java
@@ -103,12 +103,16 @@ public class MetadataIndexNode {
children.add(MetadataIndexEntry.deserializeFrom(buffer));
}
long offset = ReadWriteIOUtils.readLong(buffer);
- MetadataIndexNodeType nodeType = MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer));
+ MetadataIndexNodeType nodeType = MetadataIndexNodeType
+ .deserialize(ReadWriteIOUtils.readByte(buffer));
return new MetadataIndexNode(children, offset, nodeType);
}
- public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key) {
- int index = binarySearchInChildren(key);
+ public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key, boolean exactSearch) {
+ int index = binarySearchInChildren(key, exactSearch);
+ if (index == -1) {
+ return null;
+ }
long childEndOffset;
if (index != children.size() - 1) {
childEndOffset = children.get(index + 1).getOffset();
@@ -118,7 +122,7 @@ public class MetadataIndexNode {
return new Pair<>(children.get(index), childEndOffset);
}
- int binarySearchInChildren(String key) {
+ int binarySearchInChildren(String key, boolean exactSearch) {
int low = 0;
int high = children.size() - 1;
@@ -135,6 +139,12 @@ public class MetadataIndexNode {
return mid; // key found
}
}
- return low == 0 ? low : low - 1; // key not found
+
+ // key not found
+ if (exactSearch) {
+ return -1;
+ } else {
+ return low == 0 ? low : low - 1;
+ }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 0fa4d5a..28b2f0a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
@@ -305,13 +306,19 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (metadataIndexPair == null) {
+ return null;
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
@@ -329,7 +336,10 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
+ if (metadataIndexPair == null) {
+ return Collections.emptyList();
+ }
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
List<String> measurementList = new ArrayList<>(measurements);
Set<String> measurementsHadFound = new HashSet<>();
@@ -344,7 +354,10 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT, false);
+ }
+ if (measurementMetadataIndexPair == null) {
+ return Collections.emptyList();
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
@@ -588,7 +601,10 @@ public class TsFileSequenceReader implements AutoCloseable {
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
- metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
+ metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE, false);
+ if (metadataIndexPair == null) {
+ return Collections.emptyList();
+ }
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
generateMetadataIndex(metadataIndexPair.left, buffer, device,
@@ -611,16 +627,24 @@ public class TsFileSequenceReader implements AutoCloseable {
* return when it is not INTERNAL_MEASUREMENT. This works for the situation
* when the index tree does NOT have the device level and ONLY has the
* measurement level.
+ * @param exactSearch if is in exact search mode, return null when there is no entry with name;
+ * or else return the nearest MetadataIndexEntry before it (for deeper
+ * search)
* @return target MetadataIndexEntry, endOffset pair
*/
private Pair<MetadataIndexEntry, Long> getMetadataAndEndOffset(MetadataIndexNode metadataIndex,
- String name, MetadataIndexNodeType type) throws IOException {
- Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex.getChildIndexEntry(name);
+ String name, MetadataIndexNodeType type, boolean exactSearch) throws IOException {
+ Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex
+ .getChildIndexEntry(name, exactSearch);
+ if (childIndexEntry == null) {
+ return null;
+ }
if (!metadataIndex.getNodeType().equals(type)) {
return childIndexEntry;
}
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
- return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type);
+ return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, type,
+ exactSearch);
}
/**
@@ -990,7 +1014,7 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadata(path);
if (timeseriesMetaData == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
}
List<ChunkMetadata> chunkMetadataList = readChunkMetaDataList(timeseriesMetaData);
chunkMetadataList.sort(Comparator.comparingLong(ChunkMetadata::getStartTime));
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
index eb1ef3a..911c2f5 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java
@@ -38,9 +38,11 @@ public class MetadataIndexNodeTest {
MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L,
MetadataIndexNodeType.LEAF_MEASUREMENT);
- Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10"));
- Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13"));
- Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23"));
+ Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false));
+ Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false));
+ Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false));
+ Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s13", true));
+ Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s23", true));
}
}