You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/11/01 13:19:04 UTC
[iotdb] 01/01: [IOTDB-975] Improve readTimeseriesMetadata
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_975
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a84a66a92dbbcc90cc123db29305e2c7820af6ce
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Sun Nov 1 21:18:26 2020 +0800
[IOTDB-975] Improve readTimeseriesMetadata
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 61 ++++++----------------
1 file changed, 16 insertions(+), 45 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 2d3313f..d6e8d22 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -24,6 +24,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -330,13 +331,9 @@ public class TsFileSequenceReader implements AutoCloseable {
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
- int maxDegreeOfIndexNode = config.getMaxDegreeOfIndexNode();
- if (measurements.size() > maxDegreeOfIndexNode / Math.log(maxDegreeOfIndexNode)) {
- traverseAndReadTimeseriesMetadataInOneDevice(resultTimeseriesMetadataList,
- MetadataIndexNodeType.INTERNAL_MEASUREMENT, metadataIndexPair, measurements);
- return resultTimeseriesMetadataList;
- }
- for (String measurement : measurements) {
+ List<String> measurementList = new ArrayList<>(measurements);
+ Set<String> measurementsHadFound = new HashSet<>();
+ for (int i = 0; i < measurementList.size(); i++) {
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Pair<MetadataIndexEntry, Long> measurementMetadataIndexPair = metadataIndexPair;
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
@@ -344,54 +341,28 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
while (buffer.hasRemaining()) {
timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
}
- int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList,
- measurement);
- if (searchResult >= 0) {
- resultTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
- }
- }
- return resultTimeseriesMetadataList;
- }
-
- private void traverseAndReadTimeseriesMetadataInOneDevice(
- List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNodeType type,
- Pair<MetadataIndexEntry, Long> metadataIndexPair, Set<String> measurements)
- throws IOException {
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- switch (type) {
- case LEAF_DEVICE:
- case INTERNAL_MEASUREMENT:
- MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
- int metadataIndexListSize = metadataIndexNode.getChildren().size();
- for (int i = 0; i < metadataIndexListSize; i++) {
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexListSize - 1) {
- endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ for (int j = i; j < measurementList.size(); j++) {
+ String current = measurementList.get(j);
+ if (!measurementsHadFound.contains(current)) {
+ int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList, current);
+ if (searchResult >= 0) {
+ resultTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
+ measurementsHadFound.add(current);
}
- traverseAndReadTimeseriesMetadataInOneDevice(timeseriesMetadataList,
- metadataIndexNode.getNodeType(),
- new Pair<>(metadataIndexNode.getChildren().get(i), endOffset), measurements);
}
- break;
- case LEAF_MEASUREMENT:
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
- if (measurements.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
- }
+ if (measurementsHadFound.size() == measurements.size()) {
+ return resultTimeseriesMetadataList;
}
- break;
- default:
- throw new IOException("Failed to traverse and read TimeseriesMetadata in device: " +
- metadataIndexPair.left.getName() + ". Wrong MetadataIndexEntry type.");
+ }
}
+ return resultTimeseriesMetadataList;
}
private int binarySearchInTimeseriesMetadataList(List<TimeseriesMetadata> timeseriesMetadataList,