You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/11/03 12:32:37 UTC
[iotdb] branch new_vector updated: [IOTDB-1910] Support aligned
timeseries in TsFile metadata index (#4291)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/new_vector by this push:
new 92de56f [IOTDB-1910] Support aligned timeseries in TsFile metadata index (#4291)
92de56f is described below
commit 92de56f8cd57810c199ebd3a7a650690f3161d5c
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Wed Nov 3 20:32:08 2021 +0800
[IOTDB-1910] Support aligned timeseries in TsFile metadata index (#4291)
---
.../tsfile/file/metadata/AlignedChunkMetadata.java | 10 ---
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 11 ---
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 4 --
.../file/metadata/MetadataIndexConstructor.java | 6 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 9 ---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 35 ---------
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 4 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 61 ++--------------
...tadataTest.java => TimeseriesMetadataTest.java} | 2 +-
.../tsfile/read/TimeSeriesMetadataReadTest.java | 7 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 14 +++-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 83 ++++++++++++----------
12 files changed, 76 insertions(+), 170 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
index f42bb3c..ee4f570 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java
@@ -176,16 +176,6 @@ public class AlignedChunkMetadata implements IChunkMetadata {
return 0;
}
- @Override
- public boolean isTimeColumn() {
- return false;
- }
-
- @Override
- public boolean isValueColumn() {
- return false;
- }
-
public Chunk getTimeChunk() throws IOException {
return timeChunkMetadata.getChunkLoader().loadChunk((ChunkMetadata) timeChunkMetadata);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index dad4fdd..7e84a78 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -337,16 +336,6 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
return mask;
}
- @Override
- public boolean isTimeColumn() {
- return mask == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- @Override
- public boolean isValueColumn() {
- return mask == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setMask(byte mask) {
this.mask = mask;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
index 3893d15..1cc819f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java
@@ -74,8 +74,4 @@ public interface IChunkMetadata {
int serializeTo(OutputStream outputStream, boolean serializeStatistic) throws IOException;
byte getMask();
-
- boolean isTimeColumn();
-
- boolean isValueColumn();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
index de20f40..062ffd6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -62,21 +62,17 @@ public class MetadataIndexConstructor {
TimeseriesMetadata timeseriesMetadata;
MetadataIndexNode currentIndexNode =
new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
- int serializedTimeseriesMetadataNum = 0;
for (int i = 0; i < entry.getValue().size(); i++) {
timeseriesMetadata = entry.getValue().get(i);
- if (serializedTimeseriesMetadataNum == 0
- || serializedTimeseriesMetadataNum >= config.getMaxDegreeOfIndexNode()) {
+ if (i % config.getMaxDegreeOfIndexNode() == 0) {
if (currentIndexNode.isFull()) {
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT);
}
currentIndexNode.addEntry(
new MetadataIndexEntry(timeseriesMetadata.getMeasurementId(), out.getPosition()));
- serializedTimeseriesMetadataNum = 0;
}
timeseriesMetadata.serializeTo(out.wrapAsStream());
- serializedTimeseriesMetadataNum++;
}
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
deviceMetadataIndexMap.put(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2881711..156d220 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
@@ -148,14 +147,6 @@ public class TimeseriesMetadata implements Accountable, ITimeSeriesMetadata {
return timeSeriesMetadataType;
}
- public boolean isTimeColumn() {
- return timeSeriesMetadataType == TsFileConstant.TIME_COLUMN_MASK;
- }
-
- public boolean isValueColumn() {
- return timeSeriesMetadataType == TsFileConstant.VALUE_COLUMN_MASK;
- }
-
public void setTimeSeriesMetadataType(byte timeSeriesMetadataType) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 7588b84..c546068 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -370,41 +370,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
- /**
- * Find the leaf node that contains this vector, return all the needed subSensor and time column
- *
- * @param path path with time column
- * @param subSensorList value columns that needed
- * @return TimeseriesMetadata for the time column and all the needed subSensor, the order of the
- * element in this list should be the same as subSensorList
- */
- public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, List<String> subSensorList)
- throws IOException {
- Pair<MetadataIndexEntry, Long> metadataIndexPair = getLeafMetadataIndexPair(path);
- if (metadataIndexPair == null) {
- return Collections.emptyList();
- }
- Map<String, TimeseriesMetadata> timeseriesMetadataMap = new HashMap<>();
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
- } catch (BufferOverflowException e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
- }
- timeseriesMetadataMap.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata);
- }
-
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- for (String subSensor : subSensorList) {
- timeseriesMetadataList.add(timeseriesMetadataMap.get(subSensor));
- }
- return timeseriesMetadataList;
- }
-
/* Find the leaf node that contains path, return all the sensors in that leaf node which are also in allSensors set */
public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors)
throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
index 014d45a..a54ad6e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
@@ -554,8 +554,10 @@ public class ReadWriteIOUtils {
/** string length's type is varInt */
public static String readVarIntString(InputStream inputStream) throws IOException {
int strLength = ReadWriteForEncodingUtils.readVarInt(inputStream);
- if (strLength <= 0) {
+ if (strLength < 0) {
return null;
+ } else if (strLength == 0) {
+ return "";
}
byte[] bytes = new byte[strLength];
int readLen = inputStream.read(bytes, 0, strLength);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3759780..524cb4f5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -253,44 +253,17 @@ public class TsFileIOWriter {
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
// group ChunkMetadata by series
- // only contains ordinary path and time column of vector series
Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
- // time column -> ChunkMetadataList TreeMap of value columns in vector
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>();
-
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList();
- int idx = 0;
- while (idx < chunkMetadatas.size()) {
- IChunkMetadata chunkMetadata = chunkMetadatas.get(idx);
- if (chunkMetadata.getMask() == 0) {
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- } else if (chunkMetadata.isTimeColumn()) {
- // time column of a vector series
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector =
- vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>());
-
- // value columns of a vector series
- while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) {
- chunkMetadata = chunkMetadatas.get(idx);
- Path vectorSeries =
- new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMapInVector
- .computeIfAbsent(vectorSeries, k -> new ArrayList<>())
- .add(chunkMetadata);
- idx++;
- }
- }
+ for (IChunkMetadata chunkMetadata : chunkMetadatas) {
+ Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
}
}
- MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap, vectorToPathsMap);
+ MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap);
TsFileMetadata tsFileMetaData = new TsFileMetadata();
tsFileMetaData.setMetadataIndex(metadataIndex);
tsFileMetaData.setMetaOffset(metaOffset);
@@ -330,13 +303,9 @@ public class TsFileIOWriter {
* Flush TsFileMetadata, including ChunkMetadataList and TimeseriesMetaData
*
* @param chunkMetadataListMap chunkMetadata that Path.mask == 0
- * @param vectorToPathsMap Map Path to chunkMataList, Key is Path(timeColumn) and Value is it's
- * sub chunkMetadataListMap
* @return MetadataIndexEntry list in TsFileMetadata
*/
- private MetadataIndexNode flushMetadataIndex(
- Map<Path, List<IChunkMetadata>> chunkMetadataListMap,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private MetadataIndexNode flushMetadataIndex(Map<Path, List<IChunkMetadata>> chunkMetadataListMap)
throws IOException {
// convert ChunkMetadataList to this field
@@ -344,7 +313,7 @@ public class TsFileIOWriter {
// create device -> TimeseriesMetaDataList Map
for (Map.Entry<Path, List<IChunkMetadata>> entry : chunkMetadataListMap.entrySet()) {
// for ordinary path
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
+ flushOneChunkMetadata(entry.getKey(), entry.getValue());
}
// construct TsFileMetadata and return
@@ -356,12 +325,8 @@ public class TsFileIOWriter {
*
* @param path Path of chunk
* @param chunkMetadataList List of chunkMetadata about path(previous param)
- * @param vectorToPathsMap Key is Path(timeColumn) and Value is it's sub chunkMetadataListMap
*/
- private void flushOneChunkMetadata(
- Path path,
- List<IChunkMetadata> chunkMetadataList,
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap)
+ private void flushOneChunkMetadata(Path path, List<IChunkMetadata> chunkMetadataList)
throws IOException {
// create TimeseriesMetaData
PublicBAOS publicBAOS = new PublicBAOS();
@@ -391,18 +356,6 @@ public class TsFileIOWriter {
deviceTimeseriesMetadataMap
.computeIfAbsent(path.getDevice(), k -> new ArrayList<>())
.add(timeseriesMetadata);
-
- // for VECTOR
- for (IChunkMetadata chunkMetadata : chunkMetadataList) {
- // chunkMetadata is time column of a vector series
- if (chunkMetadata.isTimeColumn()) {
- Map<Path, List<IChunkMetadata>> vectorMap = vectorToPathsMap.get(path);
- for (Map.Entry<Path, List<IChunkMetadata>> entry : vectorMap.entrySet()) {
- flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap);
- }
- }
- break;
- }
}
/**
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadataTest.java
similarity index 98%
rename from tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java
rename to tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadataTest.java
index dc78046..bee663a 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadataTest.java
@@ -33,7 +33,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-public class TimeSeriesMetadataTest {
+public class TimeseriesMetadataTest {
public static final String measurementUID = "sensor01";
public static final int typeLength = 1024;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java
index 8015284..0a26423 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TimeSeriesMetadataReadTest.java
@@ -30,6 +30,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
@@ -38,7 +39,7 @@ import java.util.Set;
public class TimeSeriesMetadataReadTest {
private static final String FILE_PATH =
- TestConstant.BASE_OUTPUT_PATH.concat("TimeSeriesMetadataReadTest.tsfile");
+ TestConstant.BASE_OUTPUT_PATH.concat("TimeseriesMetadataReadTest.tsfile");
private final TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
private int maxDegreeOfIndexNode;
@@ -54,6 +55,10 @@ public class TimeSeriesMetadataReadTest {
public void after() {
FileGenerator.after();
conf.setMaxDegreeOfIndexNode(maxDegreeOfIndexNode);
+ File file = new File(FILE_PATH);
+ if (file.exists()) {
+ file.delete();
+ }
}
@Test
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
index 457ecf8..a594994 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java
@@ -49,9 +49,15 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/** test for MetadataIndexConstructor */
public class MetadataIndexConstructorTest {
@@ -73,6 +79,10 @@ public class MetadataIndexConstructorTest {
@After
public void after() {
conf.setMaxDegreeOfIndexNode(maxDegreeOfIndexNode);
+ File file = new File(FILE_PATH);
+ if (file.exists()) {
+ file.delete();
+ }
}
/** Example 1: 5 entities with 5 measurements each */
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
index b25befb..d731120 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.tsfile.constant.TestConstant;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
-import org.apache.iotdb.tsfile.file.metadata.TimeSeriesMetadataTest;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -51,27 +50,29 @@ import java.util.Set;
public class TsFileIOWriterTest {
- private static String tsfile = TestConstant.BASE_OUTPUT_PATH.concat("tsfileIOWriterTest.tsfile");
- private static String deviceId = "device1";
- private static int chunkGroupNum = 2;
+ private static final String FILE_PATH =
+ TestConstant.BASE_OUTPUT_PATH.concat("TsFileIOWriterTest.tsfile");
+ private static final String DEVICE_1 = "device1";
+ private static final String DEVICE_2 = "device2";
+ private static final String SENSOR_1 = "sensor1";
+
+ private static final int CHUNK_GROUP_NUM = 2;
@Before
public void before() throws IOException {
- TsFileIOWriter writer = new TsFileIOWriter(new File(tsfile));
+ TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH));
// file schema
- UnaryMeasurementSchema measurementSchema = TestHelper.createSimpleMeasurementSchema("sensor01");
+ UnaryMeasurementSchema measurementSchema = TestHelper.createSimpleMeasurementSchema(SENSOR_1);
VectorMeasurementSchema vectorMeasurementSchema =
new VectorMeasurementSchema(
- "vector",
- new String[] {"s1", "s2"},
- new TSDataType[] {TSDataType.INT64, TSDataType.INT64});
+ "", new String[] {"s1", "s2"}, new TSDataType[] {TSDataType.INT64, TSDataType.INT64});
Schema schema = new Schema();
- schema.registerTimeseries(new Path(deviceId, "sensor01"), measurementSchema);
- schema.registerTimeseries(new Path(deviceId, "vector"), vectorMeasurementSchema);
-
- writeChunkGroup(writer, measurementSchema, vectorMeasurementSchema, chunkGroupNum);
+ schema.registerTimeseries(new Path(DEVICE_1, SENSOR_1), measurementSchema);
+ schema.registerTimeseries(new Path(DEVICE_2, ""), vectorMeasurementSchema);
+ writeChunkGroup(writer, measurementSchema);
+ writeVectorChunkGroup(writer, vectorMeasurementSchema);
writer.setMinPlanIndex(100);
writer.setMaxPlanIndex(10000);
writer.writePlanIndices();
@@ -81,7 +82,7 @@ public class TsFileIOWriterTest {
@After
public void after() {
- File file = new File(tsfile);
+ File file = new File(FILE_PATH);
if (file.exists()) {
file.delete();
}
@@ -89,7 +90,7 @@ public class TsFileIOWriterTest {
@Test
public void endFileTest() throws IOException {
- TsFileSequenceReader reader = new TsFileSequenceReader(tsfile);
+ TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH);
// magic_string
Assert.assertEquals(TSFileConfig.MAGIC_STRING, reader.readHeadMagic());
@@ -100,32 +101,39 @@ public class TsFileIOWriterTest {
ChunkHeader header;
ChunkGroupHeader chunkGroupHeader;
- for (int i = 0; i < chunkGroupNum; i++) {
+ for (int i = 0; i < CHUNK_GROUP_NUM; i++) {
// chunk group header
Assert.assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker());
chunkGroupHeader = reader.readChunkGroupHeader();
- Assert.assertEquals(deviceId, chunkGroupHeader.getDeviceID());
+ Assert.assertEquals(DEVICE_1, chunkGroupHeader.getDeviceID());
// ordinary chunk header
Assert.assertEquals(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, reader.readMarker());
header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER);
- Assert.assertEquals(TimeSeriesMetadataTest.measurementUID, header.getMeasurementID());
+ Assert.assertEquals(SENSOR_1, header.getMeasurementID());
+ }
+
+ for (int i = 0; i < CHUNK_GROUP_NUM; i++) {
+ // chunk group header
+ Assert.assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker());
+ chunkGroupHeader = reader.readChunkGroupHeader();
+ Assert.assertEquals(DEVICE_2, chunkGroupHeader.getDeviceID());
// vector chunk header (time)
Assert.assertEquals(
MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER | TsFileConstant.TIME_COLUMN_MASK,
reader.readMarker());
header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER);
- Assert.assertEquals("vector", header.getMeasurementID());
+ Assert.assertEquals("", header.getMeasurementID());
// vector chunk header (values)
Assert.assertEquals(
MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER | TsFileConstant.VALUE_COLUMN_MASK,
reader.readMarker());
header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER);
- Assert.assertEquals("vector.s1", header.getMeasurementID());
+ Assert.assertEquals("s1", header.getMeasurementID());
Assert.assertEquals(
MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER | TsFileConstant.VALUE_COLUMN_MASK,
reader.readMarker());
header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER);
- Assert.assertEquals("vector.s2", header.getMeasurementID());
+ Assert.assertEquals("s2", header.getMeasurementID());
}
Assert.assertEquals(MetaMarker.OPERATION_INDEX_RANGE, reader.readMarker());
@@ -143,27 +151,22 @@ public class TsFileIOWriterTest {
deviceTimeseriesMetadataMap.entrySet()) {
for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
String seriesPath = entry.getKey() + "." + timeseriesMetadata.getMeasurementId();
- Assert.assertTrue(!pathSet.contains(seriesPath));
+ Assert.assertFalse(pathSet.contains(seriesPath));
pathSet.add(seriesPath);
}
}
// FileMetaData
TsFileMetadata metaData = reader.readFileMetadata();
- Assert.assertEquals(1, metaData.getMetadataIndex().getChildren().size());
+ Assert.assertEquals(2, metaData.getMetadataIndex().getChildren().size());
}
- private void writeChunkGroup(
- TsFileIOWriter writer,
- UnaryMeasurementSchema measurementSchema,
- VectorMeasurementSchema vectorMeasurementSchema,
- int groupNum)
+ private void writeChunkGroup(TsFileIOWriter writer, UnaryMeasurementSchema measurementSchema)
throws IOException {
- for (int i = 0; i < groupNum; i++) {
+ for (int i = 0; i < CHUNK_GROUP_NUM; i++) {
// chunk group
- writer.startChunkGroup(deviceId);
- // ordinary chunk
- // chunk statistics
+ writer.startChunkGroup(DEVICE_1);
+ // ordinary chunk, chunk statistics
Statistics statistics = Statistics.getStatsByType(measurementSchema.getType());
statistics.updateStats(0L, 0L);
writer.startFlushChunk(
@@ -176,14 +179,22 @@ public class TsFileIOWriterTest {
0,
0);
writer.endCurrentChunk();
+ writer.endChunkGroup();
+ }
+ }
+
+ private void writeVectorChunkGroup(
+ TsFileIOWriter writer, VectorMeasurementSchema vectorMeasurementSchema) throws IOException {
+ for (int i = 0; i < CHUNK_GROUP_NUM; i++) {
+ // chunk group
+ writer.startChunkGroup(DEVICE_2);
// vector chunk (time)
- Statistics vectorStatistics = Statistics.getStatsByType(vectorMeasurementSchema.getType());
writer.startFlushChunk(
vectorMeasurementSchema.getMeasurementId(),
vectorMeasurementSchema.getCompressor(),
vectorMeasurementSchema.getType(),
vectorMeasurementSchema.getTimeTSEncoding(),
- vectorStatistics,
+ Statistics.getStatsByType(vectorMeasurementSchema.getType()),
0,
0,
TsFileConstant.TIME_COLUMN_MASK);
@@ -195,9 +206,7 @@ public class TsFileIOWriterTest {
vectorMeasurementSchema.getSubMeasurementsTSDataTypeList().get(j));
subStatistics.updateStats(0L, 0L);
writer.startFlushChunk(
- vectorMeasurementSchema.getMeasurementId()
- + "."
- + vectorMeasurementSchema.getSubMeasurementsList().get(j),
+ vectorMeasurementSchema.getSubMeasurementsList().get(j),
vectorMeasurementSchema.getCompressor(),
vectorMeasurementSchema.getSubMeasurementsTSDataTypeList().get(j),
vectorMeasurementSchema.getSubMeasurementsTSEncodingList().get(j),