You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/11/10 03:30:41 UTC
[iotdb] 03/03: fix aligned write
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 265538d39cee80e5c1875ff1cc7de480189e706a
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Nov 10 11:30:03 2021 +0800
fix aligned write
---
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 11 +++--
.../org/apache/iotdb/tsfile/write/WriteTest.java | 23 ++++++-----
.../write/writer/AlignedChunkWriterImplTest.java | 48 +++++++++++-----------
.../write/writer/VectorMeasurementSchemaStub.java | 17 ++++----
4 files changed, 48 insertions(+), 51 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
index 8ad44cd..b3dbf33 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
@@ -18,17 +18,16 @@
*/
package org.apache.iotdb.tsfile.write.chunk;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
public class AlignedChunkWriterImpl implements IChunkWriter {
private final TimeChunkWriter timeChunkWriter;
@@ -36,7 +35,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
private int valueIndex;
/** @param schema schema of this measurement */
- public AlignedChunkWriterImpl(IMeasurementSchema schema) {
+ public AlignedChunkWriterImpl(VectorMeasurementSchema schema) {
timeChunkWriter =
new TimeChunkWriter(
schema.getMeasurementId(),
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
index d01836a..b98418b 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
@@ -18,6 +18,15 @@
*/
package org.apache.iotdb.tsfile.write;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Random;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
@@ -33,23 +42,12 @@ import org.apache.iotdb.tsfile.utils.StringContainer;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
-
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
/** test writing processing correction combining writing process and reading process. */
public class WriteTest {
@@ -241,6 +239,9 @@ public class WriteTest {
strings = getNextRecord(lineCount, stageState);
for (String str : strings) {
TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
+ if (record.dataPointList.isEmpty()) {
+ continue;
+ }
tsFileWriter.write(record);
}
lineCount++;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java
index 495ddbe..ac05405 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java
@@ -18,6 +18,11 @@
*/
package org.apache.iotdb.tsfile.write.writer;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -26,15 +31,8 @@ import org.apache.iotdb.tsfile.utils.PublicBAOS;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
-
import org.junit.Test;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
public class AlignedChunkWriterImplTest {
@Test
@@ -50,11 +48,11 @@ public class AlignedChunkWriterImplTest {
}
chunkWriter.sealCurrentPage();
- // time chunk: 17 + 4 + 160;
- // value chunk 1: 19 + 2 + 4 + 3 + 80;
- // value chunk 2: 19 + 2 + 4 + 3 + 20;
- // value chunk 3: 20 + 4 + 7 + 20 * 8;
- assertEquals(528, chunkWriter.getSerializedChunkSize());
+ // time chunk: 7(ChunkHeader Size) + 4(PageHeader Size: uncompressedSize + compressedSize) + 160(dataSize);
+ // value chunk 1: 8(ChunkHeader Size) + 2(PageHeader Size: uncompressedSize + compressedSize) + 4(bitmap length) + 3(bitmap data) + 80(data size);
+ // value chunk 2: 8 + 2 + 4 + 3 + 20;
+ // value chunk 3: 9 + 4 + 4 + 3 + 20 * 8;
+ assertEquals(485, chunkWriter.getSerializedChunkSize());
try {
TestTsFileOutput testTsFileOutput = new TestTsFileOutput();
@@ -65,7 +63,7 @@ public class AlignedChunkWriterImplTest {
// time chunk
assertEquals(
(byte) (0x80 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER), ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(164, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.VECTOR.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -74,7 +72,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 1
assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s1", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s1", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(89, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.FLOAT.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -83,7 +81,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 2
assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s2", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s2", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(29, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.INT32.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -92,7 +90,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 2
assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s3", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s3", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(171, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.DOUBLE.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -124,11 +122,11 @@ public class AlignedChunkWriterImplTest {
}
chunkWriter.sealCurrentPage();
- // time chunk: 17 + (4 + 17 + 160) * 2
- // value chunk 1: 20 + (2 + 41 + 4 + 3 + 80) * 2
- // value chunk 2: 20 + (2 + 41 + 4 + 3 + 20) * 2
- // value chunk 3: 20 + (4 + 57 + 4 + 3 + 160) * 2
- assertEquals(1295, chunkWriter.getSerializedChunkSize());
+ // time chunk: 7 + (4 + 17 + 160) * 2
+ // value chunk 1: 9 + (2 + 41 + 4 + 3 + 80) * 2
+ // value chunk 2: 9 + (2 + 41 + 4 + 3 + 20) * 2
+ // value chunk 3: 9 + (4 + 57 + 4 + 3 + 160) * 2
+ assertEquals(1252, chunkWriter.getSerializedChunkSize());
try {
TestTsFileOutput testTsFileOutput = new TestTsFileOutput();
@@ -138,7 +136,7 @@ public class AlignedChunkWriterImplTest {
ByteBuffer buffer = ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size());
// time chunk
assertEquals((byte) (0x80 | MetaMarker.CHUNK_HEADER), ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(362, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.VECTOR.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -147,7 +145,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 1
assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s1", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s1", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(260, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.FLOAT.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -156,7 +154,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 2
assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s2", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s2", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(140, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.INT32.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
@@ -165,7 +163,7 @@ public class AlignedChunkWriterImplTest {
// value chunk 2
assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer));
- assertEquals("vectorName.s3", ReadWriteIOUtils.readVarIntString(buffer));
+ assertEquals("s3", ReadWriteIOUtils.readVarIntString(buffer));
assertEquals(456, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
assertEquals(TSDataType.DOUBLE.serialize(), ReadWriteIOUtils.readByte(buffer));
assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer));
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
index 2128a46..e0b7ba9e 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
@@ -18,25 +18,24 @@
*/
package org.apache.iotdb.tsfile.write.writer;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
-
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
-public class VectorMeasurementSchemaStub implements IMeasurementSchema {
+public class VectorMeasurementSchemaStub extends VectorMeasurementSchema {
@Override
public String getMeasurementId() {
- return "vectorName";
+ return "";
}
@Override