You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/01/10 06:49:14 UTC
[iotdb] branch master updated: [IOTDB-2322] Provide a new interface to get AlignedChunkMetadata of device (#4742)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2a9af0b [IOTDB-2322] Provide a new interface to get AlignedChunkMetadata of device (#4742)
2a9af0b is described below
commit 2a9af0bfd29e3e0903f2e338f758bd06e4c3627b
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Mon Jan 10 14:48:43 2022 +0800
[IOTDB-2322] Provide a new interface to get AlignedChunkMetadata of device (#4742)
Co-authored-by: Zesong Sun <v-...@microsoft.com>
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 87 +++++++++++++++++++++-
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 40 ++++++++++
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 9 +--
3 files changed, 127 insertions(+), 9 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 573fa6f..8666812 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
@@ -389,7 +390,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode;
- TimeseriesMetadata firstTimeseriesMetadata = null;
+ TimeseriesMetadata firstTimeseriesMetadata;
try {
// next layer MeasurementNode of the specific DeviceNode
metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
@@ -733,7 +734,7 @@ public class TsFileSequenceReader implements AutoCloseable {
Queue<Pair<String, Pair<Long, Long>>> queue = new LinkedList<>();
for (int i = 0; i < metadataIndexEntryList.size(); i++) {
MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
- long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset();
+ long endOffset = metadataIndexNode.getEndOffset();
if (i != metadataIndexEntryList.size() - 1) {
endOffset = metadataIndexEntryList.get(i + 1).getOffset();
}
@@ -905,7 +906,7 @@ public class TsFileSequenceReader implements AutoCloseable {
List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
for (int i = 0; i < metadataIndexEntryList.size(); i++) {
MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
- long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset();
+ long endOffset = metadataIndexNode.getEndOffset();
if (i != metadataIndexEntryList.size() - 1) {
endOffset = metadataIndexEntryList.get(i + 1).getOffset();
}
@@ -1459,6 +1460,86 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/**
+ * Get AlignedChunkMetadata of sensors under one device
+ *
+ * @param device device name
+ */
+ public List<AlignedChunkMetadata> getAlignedChunkMetadata(String device) throws IOException {
+ readFileMetadata();
+ MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ Pair<MetadataIndexEntry, Long> metadataIndexPair =
+ getMetadataAndEndOffset(deviceMetadataIndexNode, device, true, true);
+ if (metadataIndexPair == null) {
+ throw new IOException("Device {" + device + "} is not in tsFileMetaData");
+ }
+ ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ MetadataIndexNode metadataIndexNode;
+ TimeseriesMetadata firstTimeseriesMetadata;
+ try {
+ // next layer MeasurementNode of the specific DeviceNode
+ metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer);
+ } catch (BufferOverflowException e) {
+ logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
+ throw e;
+ }
+ firstTimeseriesMetadata = tryToGetFirstTimeseriesMetadata(metadataIndexNode);
+ if (firstTimeseriesMetadata == null) {
+ throw new IOException("Timeseries of device {" + device + "} are not aligned");
+ }
+
+ Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
+ List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+
+ for (int i = 0; i < metadataIndexEntryList.size(); i++) {
+ MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexEntryList.size() - 1) {
+ endOffset = metadataIndexEntryList.get(i + 1).getOffset();
+ }
+ buffer = readData(metadataIndexEntry.getOffset(), endOffset);
+ if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ while (buffer.hasRemaining()) {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
+ }
+ timeseriesMetadataMap
+ .computeIfAbsent(device, k -> new ArrayList<>())
+ .addAll(timeseriesMetadataList);
+ } else {
+ generateMetadataIndex(
+ metadataIndexEntry,
+ buffer,
+ device,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ true);
+ }
+ }
+
+ for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) {
+ TimeseriesMetadata timeseriesMetadata = timeseriesMetadataList.get(0);
+ List<TimeseriesMetadata> valueTimeseriesMetadataList = new ArrayList<>();
+
+ for (int i = 1; i < timeseriesMetadataList.size(); i++) {
+ valueTimeseriesMetadataList.add(timeseriesMetadataList.get(i));
+ }
+
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata =
+ new AlignedTimeSeriesMetadata(timeseriesMetadata, valueTimeseriesMetadataList);
+ List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
+ for (IChunkMetadata chunkMetadata : readIChunkMetaDataList(alignedTimeSeriesMetadata)) {
+ chunkMetadataList.add((AlignedChunkMetadata) chunkMetadata);
+ }
+ // only one timeseriesMetadataList in one device
+ return chunkMetadataList;
+ }
+
+ throw new IOException(
+ String.format(
+ "Error when reading timeseriesMetadata of device %s in file %s", device, file));
+ }
+
+ /**
* get ChunkMetaDatas in given TimeseriesMetaData
*
* @return List of ChunkMetaData
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileReaderTest.java
index 7a0f932..c80aeec 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileReaderTest.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.tsfile.read;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -51,6 +53,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
public class TsFileReaderTest {
@@ -474,4 +477,41 @@ public class TsFileReaderTest {
}
return rowCount;
}
+
+ @Test
+ public void testGetAlignedChunkMetadata() throws IOException {
+ // generate aligned timeseries "d1.s1","d1.s2","d1.s3","d1.s4" and nonAligned timeseries
+ // "d2.s1","d2.s2","d2.s3"
+ TsFileGeneratorForTest.generateAlignedTsFile(10, 100, 30);
+ String filePath = TsFileGeneratorForTest.alignedOutputDataFile;
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
+ // query for non-exist device
+ try {
+ reader.getAlignedChunkMetadata("d3");
+ } catch (IOException e) {
+ Assert.assertEquals("Device {d3} is not in tsFileMetaData", e.getMessage());
+ }
+
+ // query for non-aligned device
+ try {
+ reader.getAlignedChunkMetadata("d2");
+ } catch (IOException e) {
+ Assert.assertEquals("Timeseries of device {d2} are not aligned", e.getMessage());
+ }
+
+ String[] expected = new String[] {"s1", "s2", "s3", "s4"};
+
+ List<AlignedChunkMetadata> chunkMetadataList = reader.getAlignedChunkMetadata("d1");
+ AlignedChunkMetadata alignedChunkMetadata = chunkMetadataList.get(0);
+ Assert.assertEquals("", alignedChunkMetadata.getTimeChunkMetadata().getMeasurementUid());
+ int i = 0;
+ for (IChunkMetadata chunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) {
+ Assert.assertEquals(expected[i], chunkMetadata.getMeasurementUid());
+ i++;
+ }
+
+ Assert.assertEquals(expected.length, i);
+ }
+ TsFileGeneratorForTest.closeAlignedTsFile();
+ }
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
index 5779896..3f3b199 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
@@ -116,10 +116,9 @@ public class TsFileGeneratorForTest {
Assert.assertTrue(file.delete());
}
file.getParentFile().mkdirs();
- FileWriter fw = new FileWriter(file);
- long startTime = START_TIMESTAMP;
- try {
+ try (FileWriter fw = new FileWriter(file)) {
+ long startTime = START_TIMESTAMP;
for (int i = 0; i < maxRowCount; i++) {
// write d1
String d1 = "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," + (i * 10 + 2);
@@ -150,7 +149,7 @@ public class TsFileGeneratorForTest {
d2 += ",s1," + (i * 10 + 1);
}
if (i % 8 == 0) {
- d2 += ",s4," + "dog" + i % 4;
+ d2 += ",s4," + "dog" + 0;
}
fw.write(d2 + "\r\n");
}
@@ -165,8 +164,6 @@ public class TsFileGeneratorForTest {
fw.write(d + "\r\n");
d = "d2," + (startTime + rowCount + 1) + ",2,s-1," + (rowCount * 10 + 2);
fw.write(d + "\r\n");
- } finally {
- fw.close();
}
}