You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2021/12/28 04:44:16 UTC
[iotdb] branch jira_2217 created (now 7a17553)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch jira_2217
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 7a17553 Fix to iterator
This branch includes the following new commits:
new 9ab5e88 Add iterator to return timeseries Path in dictionary order
new 7a17553 Fix to iterator
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: Fix to iterator
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_2217
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7a17553cbeb43b11f347b7d951e9db72b6318f59
Author: Zesong Sun <v-...@microsoft.com>
AuthorDate: Tue Dec 28 12:40:16 2021 +0800
Fix to iterator
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 83 +++++++++++++++-------
.../tsfile/write/MetadataIndexConstructorTest.java | 16 ++---
2 files changed, 66 insertions(+), 33 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 59b777a..fd34be0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -649,19 +649,32 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
- * this function return all timeseries names in dictionary order
+ * this function return all timeseries names
*
* @return list of Paths
* @throws IOException io error
*/
public List<Path> getAllPaths() throws IOException {
- if (tsFileMetaData == null) {
- readFileMetadata();
- }
List<Path> paths = new ArrayList<>();
+ for (String device : getAllDevices()) {
+ Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(device);
+ for (String measurementId : timeseriesMetadataMap.keySet()) {
+ paths.add(new Path(device, measurementId));
+ }
+ }
+ return paths;
+ }
+
+ /**
+ * @return an iterator of timeseries list, in which names of timeseries are ordered in dictionary order
+ * @throws IOException io error
+ */
+ public Iterator<List<Path>> getPathsIterator() throws IOException {
+ readFileMetadata();
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+ Queue<Pair<String, Pair<Long, Long>>> queue = new LinkedList<>();
for (int i = 0; i < metadataIndexEntryList.size(); i++) {
MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset();
@@ -674,10 +687,37 @@ public class TsFileSequenceReader implements AutoCloseable {
buffer,
null,
metadataIndexNode.getNodeType(),
- paths,
- false);
+ queue);
}
- return paths;
+ return new Iterator<List<Path>>() {
+ @Override
+ public boolean hasNext() {
+ return !queue.isEmpty();
+ }
+
+ @Override
+ public List<Path> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ Pair<String, Pair<Long, Long>> startEndPair = queue.remove();
+ List<Path> paths = new ArrayList<>();
+ try {
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ ByteBuffer nextBuffer = readData(startEndPair.right.left, startEndPair.right.right);
+ while (nextBuffer.hasRemaining()) {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(nextBuffer, false));
+ }
+ for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) {
+ paths.add(new Path(startEndPair.left, timeseriesMetadata.getMeasurementId()));
+ }
+ return paths;
+ } catch (IOException e) {
+ throw new TsFileRuntimeException(
+ "Error occurred while reading a time series metadata block.");
+ }
+ }
+ };
}
private void getAllPaths(
@@ -685,30 +725,24 @@ public class TsFileSequenceReader implements AutoCloseable {
ByteBuffer buffer,
String deviceId,
MetadataIndexNodeType type,
- List<Path> timeseriesMetadataMap,
- boolean needChunkMetadata)
+ Queue<Pair<String, Pair<Long, Long>>> queue)
throws IOException {
try {
- if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- while (buffer.hasRemaining()) {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata));
- }
- for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) {
- timeseriesMetadataMap.add(new Path(deviceId, timeseriesMetadata.getMeasurementId()));
- }
- } else {
- // deviceId should be determined by LEAF_DEVICE node
- if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) {
- deviceId = metadataIndex.getName();
- }
+ if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) {
+ deviceId = metadataIndex.getName();
+ }
MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
+ long startOffset = metadataIndexNode.getChildren().get(i).getOffset();
long endOffset = metadataIndexNode.getEndOffset();
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
+ if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ queue.add(new Pair<>(deviceId, new Pair<>(startOffset, endOffset)));
+ continue;
+ }
ByteBuffer nextBuffer =
readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
getAllPaths(
@@ -716,9 +750,8 @@ public class TsFileSequenceReader implements AutoCloseable {
nextBuffer,
deviceId,
metadataIndexNode.getNodeType(),
- timeseriesMetadataMap,
- needChunkMetadata);
- }
+ queue);
+
}
} catch (BufferOverflowException e) {
logger.error("Something error happened while getting all paths of file {}", file);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
index 5d33a84..1922445 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
@@ -49,11 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -240,9 +236,13 @@ public class MetadataIndexConstructorTest {
}
}
try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) {
- List<Path> actualPaths = reader.getAllPaths();
- for (int i = 0; i < actualPaths.size(); i++) {
- assertEquals(actualPaths.get(i).getFullPath(), correctPaths.get(i));
+ Iterator<List<Path>> iterator = reader.getPathsIterator();
+ int idx = 0;
+ while (iterator.hasNext()) {
+ for (Path actualPath : iterator.next()) {
+ assertEquals(actualPath.getFullPath(), correctPaths.get(idx));
+ idx++;
+ }
}
} catch (IOException e) {
e.printStackTrace();
[iotdb] 01/02: Add iterator to return timeseries Path in dictionary order
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_2217
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9ab5e8857053b819d0bd1fabb910be176b50d82b
Author: Zesong Sun <v-...@microsoft.com>
AuthorDate: Tue Dec 28 11:09:48 2021 +0800
Add iterator to return timeseries Path in dictionary order
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 71 ++++++++++++++++++++--
.../tsfile/write/MetadataIndexConstructorTest.java | 23 ++++++-
2 files changed, 86 insertions(+), 8 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index bfb8e31..59b777a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -649,22 +649,83 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
- * this function return all timeseries names in this file
+ * this function return all timeseries names in dictionary order
*
* @return list of Paths
* @throws IOException io error
*/
public List<Path> getAllPaths() throws IOException {
+ if (tsFileMetaData == null) {
+ readFileMetadata();
+ }
List<Path> paths = new ArrayList<>();
- for (String device : getAllDevices()) {
- Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(device);
- for (String measurementId : timeseriesMetadataMap.keySet()) {
- paths.add(new Path(device, measurementId));
+
+ MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
+ List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+ for (int i = 0; i < metadataIndexEntryList.size(); i++) {
+ MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
+ long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset();
+ if (i != metadataIndexEntryList.size() - 1) {
+ endOffset = metadataIndexEntryList.get(i + 1).getOffset();
}
+ ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
+ getAllPaths(
+ metadataIndexEntry,
+ buffer,
+ null,
+ metadataIndexNode.getNodeType(),
+ paths,
+ false);
}
return paths;
}
+ private void getAllPaths(
+ MetadataIndexEntry metadataIndex,
+ ByteBuffer buffer,
+ String deviceId,
+ MetadataIndexNodeType type,
+ List<Path> timeseriesMetadataMap,
+ boolean needChunkMetadata)
+ throws IOException {
+ try {
+ if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ while (buffer.hasRemaining()) {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata));
+ }
+ for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) {
+ timeseriesMetadataMap.add(new Path(deviceId, timeseriesMetadata.getMeasurementId()));
+ }
+ } else {
+ // deviceId should be determined by LEAF_DEVICE node
+ if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) {
+ deviceId = metadataIndex.getName();
+ }
+ MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
+ int metadataIndexListSize = metadataIndexNode.getChildren().size();
+ for (int i = 0; i < metadataIndexListSize; i++) {
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexListSize - 1) {
+ endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ }
+ ByteBuffer nextBuffer =
+ readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
+ getAllPaths(
+ metadataIndexNode.getChildren().get(i),
+ nextBuffer,
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
+ }
+ } catch (BufferOverflowException e) {
+ logger.error("Something error happened while getting all paths of file {}", file);
+ throw e;
+ }
+ }
+
private TimeseriesMetadata tryToGetFirstTimeseriesMetadata(MetadataIndexNode measurementNode)
throws IOException {
// Not aligned timeseries
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
index ee73f8f..5d33a84 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
@@ -151,7 +151,7 @@ public class MetadataIndexConstructorTest {
int[][] vectorMeasurement = new int[deviceNum][];
String[][] singleMeasurement = new String[deviceNum][];
for (int i = 0; i < deviceNum; i++) {
- devices[i] = "d" + i;
+ devices[i] = "d" + generateIndexString(i, deviceNum);
vectorMeasurement[i] = new int[0];
singleMeasurement[i] = new String[measurementNum];
for (int j = 0; j < measurementNum; j++) {
@@ -204,8 +204,9 @@ public class MetadataIndexConstructorTest {
List<String> correctDevices = new ArrayList<>(); // contains all device by sequence
List<List<String>> correctFirstMeasurements =
new ArrayList<>(); // contains first measurements of every leaf, group by device
+ List<String> correctPaths = new ArrayList<>(); // contains all paths by sequence
generateCorrectResult(
- correctDevices, correctFirstMeasurements, devices, vectorMeasurement, singleMeasurement);
+ correctDevices, correctFirstMeasurements, correctPaths, devices, vectorMeasurement, singleMeasurement);
// 4. compare correct result with TsFile's metadata
Arrays.sort(devices);
// 4.1 make sure device in order
@@ -229,6 +230,7 @@ public class MetadataIndexConstructorTest {
e.printStackTrace();
fail(e.getMessage());
}
+
// 4.3 make sure split leaf correctly
for (int j = 0; j < actualDevices.size(); j++) {
for (int i = 0; i < actualMeasurements.get(j).size(); i++) {
@@ -237,6 +239,15 @@ public class MetadataIndexConstructorTest {
correctFirstMeasurements.get(j).get(i * conf.getMaxDegreeOfIndexNode()));
}
}
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) {
+ List<Path> actualPaths = reader.getAllPaths();
+ for (int i = 0; i < actualPaths.size(); i++) {
+ assertEquals(actualPaths.get(i).getFullPath(), correctPaths.get(i));
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
}
/**
@@ -331,6 +342,7 @@ public class MetadataIndexConstructorTest {
private void generateCorrectResult(
List<String> correctDevices,
List<List<String>> correctMeasurements,
+ List<String> correctPaths,
String[] devices,
int[][] vectorMeasurement,
String[][] singleMeasurement) {
@@ -341,16 +353,21 @@ public class MetadataIndexConstructorTest {
List<String> measurements = new ArrayList<>();
// single-variable measurement
if (singleMeasurement != null) {
- measurements.addAll(Arrays.asList(singleMeasurement[i]));
+ for (String measurement : singleMeasurement[i]) {
+ measurements.add(measurement);
+ correctPaths.add(new Path(device, measurement).getFullPath());
+ }
}
// multi-variable measurement
for (int vectorIndex = 0; vectorIndex < vectorMeasurement[i].length; vectorIndex++) {
measurements.add("");
+ correctPaths.add(new Path(device, "").getFullPath());
int measurementNum = vectorMeasurement[i][vectorIndex];
for (int measurementIndex = 0; measurementIndex < measurementNum; measurementIndex++) {
String measurementName =
measurementPrefix + generateIndexString(measurementIndex, measurementNum);
measurements.add(TsFileConstant.PATH_SEPARATOR + measurementName);
+ correctPaths.add(new Path(device, measurementName).getFullPath());
}
}
Collections.sort(measurements);