You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/11/01 13:19:03 UTC
[iotdb] branch jira_975 created (now a84a66a)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch jira_975
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at a84a66a [IOTDB-975] Improve readTimeseriesMetadata
This branch includes the following new commits:
new a84a66a [IOTDB-975] Improve readTimeseriesMetadata
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-975] Improve readTimeseriesMetadata
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_975
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a84a66a92dbbcc90cc123db29305e2c7820af6ce
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Sun Nov 1 21:18:26 2020 +0800
[IOTDB-975] Improve readTimeseriesMetadata
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 61 ++++++----------------
1 file changed, 16 insertions(+), 45 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 2d3313f..d6e8d22 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -24,6 +24,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -330,13 +331,9 @@ public class TsFileSequenceReader implements AutoCloseable {
Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset(
deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
- int maxDegreeOfIndexNode = config.getMaxDegreeOfIndexNode();
- if (measurements.size() > maxDegreeOfIndexNode / Math.log(maxDegreeOfIndexNode)) {
- traverseAndReadTimeseriesMetadataInOneDevice(resultTimeseriesMetadataList,
- MetadataIndexNodeType.INTERNAL_MEASUREMENT, metadataIndexPair, measurements);
- return resultTimeseriesMetadataList;
- }
- for (String measurement : measurements) {
+ List<String> measurementList = new ArrayList<>(measurements);
+ Set<String> measurementsHadFound = new HashSet<>();
+ for (int i = 0; i < measurementList.size(); i++) {
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
Pair<MetadataIndexEntry, Long> measurementMetadataIndexPair = metadataIndexPair;
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
@@ -344,54 +341,28 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode,
- measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT);
+ measurementList.get(i), MetadataIndexNodeType.INTERNAL_MEASUREMENT);
}
buffer = readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
while (buffer.hasRemaining()) {
timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
}
- int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList,
- measurement);
- if (searchResult >= 0) {
- resultTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
- }
- }
- return resultTimeseriesMetadataList;
- }
-
- private void traverseAndReadTimeseriesMetadataInOneDevice(
- List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNodeType type,
- Pair<MetadataIndexEntry, Long> metadataIndexPair, Set<String> measurements)
- throws IOException {
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- switch (type) {
- case LEAF_DEVICE:
- case INTERNAL_MEASUREMENT:
- MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
- int metadataIndexListSize = metadataIndexNode.getChildren().size();
- for (int i = 0; i < metadataIndexListSize; i++) {
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexListSize - 1) {
- endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ for (int j = i; j < measurementList.size(); j++) {
+ String current = measurementList.get(j);
+ if (!measurementsHadFound.contains(current)) {
+ int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList, current);
+ if (searchResult >= 0) {
+ resultTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
+ measurementsHadFound.add(current);
}
- traverseAndReadTimeseriesMetadataInOneDevice(timeseriesMetadataList,
- metadataIndexNode.getNodeType(),
- new Pair<>(metadataIndexNode.getChildren().get(i), endOffset), measurements);
}
- break;
- case LEAF_MEASUREMENT:
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer);
- if (measurements.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
- }
+ if (measurementsHadFound.size() == measurements.size()) {
+ return resultTimeseriesMetadataList;
}
- break;
- default:
- throw new IOException("Failed to traverse and read TimeseriesMetadata in device: " +
- metadataIndexPair.left.getName() + ". Wrong MetadataIndexEntry type.");
+ }
}
+ return resultTimeseriesMetadataList;
}
private int binarySearchInTimeseriesMetadataList(List<TimeseriesMetadata> timeseriesMetadataList,