You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/04/14 14:04:45 UTC
[iotdb] branch master updated: [IOTDB-2909] ColumnEncoder(s) for BOOLEAN and TEXT (#5512)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3309707399 [IOTDB-2909] ColumnEncoder(s) for BOOLEAN and TEXT (#5512)
3309707399 is described below
commit 330970739956a7c71078215d58e5e699a4a397c4
Author: Zhong Wang <wa...@alibaba-inc.com>
AuthorDate: Thu Apr 14 22:04:38 2022 +0800
[IOTDB-2909] ColumnEncoder(s) for BOOLEAN and TEXT (#5512)
---
...nEncoder.java => BinaryArrayColumnEncoder.java} | 43 +++++-------
...umnEncoder.java => ByteArrayColumnEncoder.java} | 36 ++--------
.../read/common/block/column/ColumnEncoder.java | 78 +++++++++++---------
.../common/block/column/ColumnEncoderFactory.java | 2 +
.../block/column/Int32ArrayColumnEncoder.java | 11 ++-
.../block/column/Int64ArrayColumnEncoder.java | 11 ++-
.../common/block/BinaryArrayColumnEncoderTest.java | 82 ++++++++++++++++++++++
.../common/block/ByteArrayColumnEncoderTest.java | 78 ++++++++++++++++++++
.../common/block/Int32ArrayColumnEncoderTest.java | 29 +++++---
.../common/block/Int64ArrayColumnEncoderTest.java | 29 +++++---
.../tsfile/common/block/TsBlockSerdeTest.java | 17 ++++-
11 files changed, 293 insertions(+), 123 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
similarity index 70%
copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
copy to tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
index 4e0bfed482..9feeaa0197 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
@@ -20,41 +20,40 @@
package org.apache.iotdb.tsfile.read.common.block.column;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Binary;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.FLOAT;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.INT32;
-
-public class Int32ArrayColumnEncoder implements ColumnEncoder {
+public class BinaryArrayColumnEncoder implements ColumnEncoder {
@Override
public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int positionCount) {
-
// Serialized data layout:
// +---------------+-----------------+-------------+
// | may have null | null indicators | values |
// +---------------+-----------------+-------------+
- // | byte | list[byte] | list[int32] |
+ // | byte | list[byte] | list[entry] |
// +---------------+-----------------+-------------+
+ //
+ // Each entry is represented as:
+ // +---------------+-------+
+ // | value length | value |
+ // +---------------+-------+
+ // | int32 | bytes |
+ // +---------------+-------+
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount);
TSDataType dataType = columnBuilder.getDataType();
- if (INT32.equals(dataType)) {
- for (int i = 0; i < positionCount; i++) {
- if (nullIndicators == null || !nullIndicators[i]) {
- columnBuilder.writeInt(input.getInt());
- } else {
- columnBuilder.appendNull();
- }
- }
- } else if (FLOAT.equals(dataType)) {
+ if (TSDataType.TEXT.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
- columnBuilder.writeFloat(Float.intBitsToFloat(input.getInt()));
+ int length = input.getInt();
+ byte[] value = new byte[length];
+ input.get(value);
+ columnBuilder.writeBinary(new Binary(value));
} else {
columnBuilder.appendNull();
}
@@ -71,16 +70,12 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
TSDataType dataType = column.getDataType();
int positionCount = column.getPositionCount();
- if (INT32.equals(dataType)) {
- for (int i = 0; i < positionCount; i++) {
- if (!column.isNull(i)) {
- output.writeInt(column.getInt(i));
- }
- }
- } else if (FLOAT.equals(dataType)) {
+ if (TSDataType.TEXT.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (!column.isNull(i)) {
- output.writeInt(Float.floatToIntBits(column.getFloat(i)));
+ Binary binary = column.getBinary(i);
+ output.writeInt(binary.getLength());
+ output.write(binary.getValues());
}
}
} else {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
similarity index 66%
copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
copy to tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
index 4e0bfed482..75bf1df701 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ByteArrayColumnEncoder.java
@@ -25,10 +25,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.FLOAT;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.INT32;
-
-public class Int32ArrayColumnEncoder implements ColumnEncoder {
+public class ByteArrayColumnEncoder implements ColumnEncoder {
@Override
public void readColumn(ColumnBuilder columnBuilder, ByteBuffer input, int positionCount) {
@@ -37,24 +34,16 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
// +---------------+-----------------+-------------+
// | may have null | null indicators | values |
// +---------------+-----------------+-------------+
- // | byte | list[byte] | list[int32] |
+ // | byte | list[byte] | list[byte] |
// +---------------+-----------------+-------------+
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount);
-
TSDataType dataType = columnBuilder.getDataType();
- if (INT32.equals(dataType)) {
- for (int i = 0; i < positionCount; i++) {
- if (nullIndicators == null || !nullIndicators[i]) {
- columnBuilder.writeInt(input.getInt());
- } else {
- columnBuilder.appendNull();
- }
- }
- } else if (FLOAT.equals(dataType)) {
+ if (TSDataType.BOOLEAN.equals(dataType)) {
+ boolean[] values = ColumnEncoder.deserializeBooleanArray(input, positionCount);
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
- columnBuilder.writeFloat(Float.intBitsToFloat(input.getInt()));
+ columnBuilder.writeBoolean(values[i]);
} else {
columnBuilder.appendNull();
}
@@ -70,19 +59,8 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
ColumnEncoder.serializeNullIndicators(output, column);
TSDataType dataType = column.getDataType();
- int positionCount = column.getPositionCount();
- if (INT32.equals(dataType)) {
- for (int i = 0; i < positionCount; i++) {
- if (!column.isNull(i)) {
- output.writeInt(column.getInt(i));
- }
- }
- } else if (FLOAT.equals(dataType)) {
- for (int i = 0; i < positionCount; i++) {
- if (!column.isNull(i)) {
- output.writeInt(Float.floatToIntBits(column.getFloat(i)));
- }
- }
+ if (TSDataType.BOOLEAN.equals(dataType)) {
+ ColumnEncoder.serializeBooleanArray(output, column, Column::getBoolean);
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
index 5520437eff..e24639549c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoder.java
@@ -37,21 +37,38 @@ public interface ColumnEncoder {
if (!mayHaveNull) {
return;
}
+ serializeBooleanArray(output, column, Column::isNull);
+ }
+
+ static boolean[] deserializeNullIndicators(ByteBuffer input, int positionCount) {
+ boolean mayHaveNull = input.get() != 0;
+ if (!mayHaveNull) {
+ return null;
+ }
+ return deserializeBooleanArray(input, positionCount);
+ }
+
+ interface ColumnToBooleanFunction {
+ boolean apply(Column column, int position);
+ }
+ static void serializeBooleanArray(
+ DataOutputStream output, Column column, ColumnToBooleanFunction toBooleanFunction)
+ throws IOException {
int positionCount = column.getPositionCount();
byte[] packedIsNull = new byte[((positionCount & ~0b111) + 1) / 8];
int currentByte = 0;
for (int position = 0; position < (positionCount & ~0b111); position += 8, currentByte++) {
byte value = 0;
- value |= column.isNull(position) ? 0b1000_0000 : 0;
- value |= column.isNull(position + 1) ? 0b0100_0000 : 0;
- value |= column.isNull(position + 2) ? 0b0010_0000 : 0;
- value |= column.isNull(position + 3) ? 0b0001_0000 : 0;
- value |= column.isNull(position + 4) ? 0b0000_1000 : 0;
- value |= column.isNull(position + 5) ? 0b0000_0100 : 0;
- value |= column.isNull(position + 6) ? 0b0000_0010 : 0;
- value |= column.isNull(position + 7) ? 0b0000_0001 : 0;
+ value |= toBooleanFunction.apply(column, position) ? 0b1000_0000 : 0;
+ value |= toBooleanFunction.apply(column, position + 1) ? 0b0100_0000 : 0;
+ value |= toBooleanFunction.apply(column, position + 2) ? 0b0010_0000 : 0;
+ value |= toBooleanFunction.apply(column, position + 3) ? 0b0001_0000 : 0;
+ value |= toBooleanFunction.apply(column, position + 4) ? 0b0000_1000 : 0;
+ value |= toBooleanFunction.apply(column, position + 5) ? 0b0000_0100 : 0;
+ value |= toBooleanFunction.apply(column, position + 6) ? 0b0000_0010 : 0;
+ value |= toBooleanFunction.apply(column, position + 7) ? 0b0000_0001 : 0;
packedIsNull[currentByte] = value;
}
@@ -62,47 +79,42 @@ public interface ColumnEncoder {
byte value = 0;
int mask = 0b1000_0000;
for (int position = positionCount & ~0b111; position < positionCount; position++) {
- value |= column.isNull(position) ? mask : 0;
+ value |= toBooleanFunction.apply(column, position) ? mask : 0;
mask >>>= 1;
}
output.write(value);
}
}
- static boolean[] deserializeNullIndicators(ByteBuffer input, int positionCount) {
- boolean mayHaveNull = input.get() != 0;
- if (!mayHaveNull) {
- return null;
- }
-
- byte[] packedIsNull = new byte[(positionCount + 7) / 8];
- input.get(packedIsNull);
+ static boolean[] deserializeBooleanArray(ByteBuffer input, int size) {
+ byte[] packedBooleanArray = new byte[(size + 7) / 8];
+ input.get(packedBooleanArray);
// read null bits 8 at a time
- boolean[] valueIsNull = new boolean[positionCount];
+ boolean[] output = new boolean[size];
int currentByte = 0;
- for (int position = 0; position < (positionCount & ~0b111); position += 8, currentByte++) {
- byte value = packedIsNull[currentByte];
- valueIsNull[position] = ((value & 0b1000_0000) != 0);
- valueIsNull[position + 1] = ((value & 0b0100_0000) != 0);
- valueIsNull[position + 2] = ((value & 0b0010_0000) != 0);
- valueIsNull[position + 3] = ((value & 0b0001_0000) != 0);
- valueIsNull[position + 4] = ((value & 0b0000_1000) != 0);
- valueIsNull[position + 5] = ((value & 0b0000_0100) != 0);
- valueIsNull[position + 6] = ((value & 0b0000_0010) != 0);
- valueIsNull[position + 7] = ((value & 0b0000_0001) != 0);
+ for (int position = 0; position < (size & ~0b111); position += 8, currentByte++) {
+ byte value = packedBooleanArray[currentByte];
+ output[position] = ((value & 0b1000_0000) != 0);
+ output[position + 1] = ((value & 0b0100_0000) != 0);
+ output[position + 2] = ((value & 0b0010_0000) != 0);
+ output[position + 3] = ((value & 0b0001_0000) != 0);
+ output[position + 4] = ((value & 0b0000_1000) != 0);
+ output[position + 5] = ((value & 0b0000_0100) != 0);
+ output[position + 6] = ((value & 0b0000_0010) != 0);
+ output[position + 7] = ((value & 0b0000_0001) != 0);
}
// read last null bits
- if ((positionCount & 0b111) > 0) {
- byte value = packedIsNull[packedIsNull.length - 1];
+ if ((size & 0b111) > 0) {
+ byte value = packedBooleanArray[packedBooleanArray.length - 1];
int mask = 0b1000_0000;
- for (int position = positionCount & ~0b111; position < positionCount; position++) {
- valueIsNull[position] = ((value & mask) != 0);
+ for (int position = size & ~0b111; position < size; position++) {
+ output[position] = ((value & mask) != 0);
mask >>>= 1;
}
}
- return valueIsNull;
+ return output;
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
index e45af001d7..4c08a17686 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnEncoderFactory.java
@@ -29,6 +29,8 @@ public class ColumnEncoderFactory {
static {
encodingToEncoder.put(ColumnEncoding.INT32_ARRAY, new Int32ArrayColumnEncoder());
encodingToEncoder.put(ColumnEncoding.INT64_ARRAY, new Int64ArrayColumnEncoder());
+ encodingToEncoder.put(ColumnEncoding.BYTE_ARRAY, new ByteArrayColumnEncoder());
+ encodingToEncoder.put(ColumnEncoding.BINARY_ARRAY, new BinaryArrayColumnEncoder());
}
public static ColumnEncoder get(ColumnEncoding columnEncoding) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
index 4e0bfed482..da95bd8cae 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
@@ -25,9 +25,6 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.FLOAT;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.INT32;
-
public class Int32ArrayColumnEncoder implements ColumnEncoder {
@Override
@@ -43,7 +40,7 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount);
TSDataType dataType = columnBuilder.getDataType();
- if (INT32.equals(dataType)) {
+ if (TSDataType.INT32.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeInt(input.getInt());
@@ -51,7 +48,7 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
columnBuilder.appendNull();
}
}
- } else if (FLOAT.equals(dataType)) {
+ } else if (TSDataType.FLOAT.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeFloat(Float.intBitsToFloat(input.getInt()));
@@ -71,13 +68,13 @@ public class Int32ArrayColumnEncoder implements ColumnEncoder {
TSDataType dataType = column.getDataType();
int positionCount = column.getPositionCount();
- if (INT32.equals(dataType)) {
+ if (TSDataType.INT32.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (!column.isNull(i)) {
output.writeInt(column.getInt(i));
}
}
- } else if (FLOAT.equals(dataType)) {
+ } else if (TSDataType.FLOAT.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (!column.isNull(i)) {
output.writeInt(Float.floatToIntBits(column.getFloat(i)));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
index bfff7aed9d..a7f1154d4a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
@@ -25,9 +25,6 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.DOUBLE;
-import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.INT64;
-
public class Int64ArrayColumnEncoder implements ColumnEncoder {
@Override
@@ -43,7 +40,7 @@ public class Int64ArrayColumnEncoder implements ColumnEncoder {
boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount);
TSDataType dataType = columnBuilder.getDataType();
- if (INT64.equals(dataType)) {
+ if (TSDataType.INT64.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeLong(input.getLong());
@@ -51,7 +48,7 @@ public class Int64ArrayColumnEncoder implements ColumnEncoder {
columnBuilder.appendNull();
}
}
- } else if (DOUBLE.equals(dataType)) {
+ } else if (TSDataType.DOUBLE.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (nullIndicators == null || !nullIndicators[i]) {
columnBuilder.writeDouble(Double.longBitsToDouble(input.getLong()));
@@ -71,13 +68,13 @@ public class Int64ArrayColumnEncoder implements ColumnEncoder {
TSDataType dataType = column.getDataType();
int positionCount = column.getPositionCount();
- if (INT64.equals(dataType)) {
+ if (TSDataType.INT64.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (!column.isNull(i)) {
output.writeLong(column.getLong(i));
}
}
- } else if (DOUBLE.equals(dataType)) {
+ } else if (TSDataType.DOUBLE.equals(dataType)) {
for (int i = 0; i < positionCount; i++) {
if (!column.isNull(i)) {
output.writeLong(Double.doubleToLongBits(column.getDouble(i)));
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
new file mode 100644
index 0000000000..4a63caa9bf
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/BinaryArrayColumnEncoderTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.common.block;
+
+import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.BinaryColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Optional;
+import java.util.Random;
+
+public class BinaryArrayColumnEncoderTest {
+ @Test
+ public void testBinaryColumn() {
+ final int positionCount = 10;
+
+ Random random = new Random();
+
+ boolean[] nullIndicators = new boolean[positionCount];
+ Binary[] values = new Binary[positionCount];
+ for (int i = 0; i < positionCount; i++) {
+ nullIndicators[i] = i % 2 == 0;
+ if (i % 2 != 0) {
+ int length = random.nextInt(1024) + 1;
+ byte[] value = new byte[length];
+ random.nextBytes(value);
+ values[i] = new Binary(value);
+ }
+ }
+ BinaryColumn input = new BinaryColumn(positionCount, Optional.of(nullIndicators), values);
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.BINARY_ARRAY);
+
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ try {
+ encoder.writeColumn(dos, input);
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+ BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null, positionCount);
+ encoder.readColumn(binaryColumnBuilder, buffer, positionCount);
+ BinaryColumn output = (BinaryColumn) binaryColumnBuilder.build();
+ Assert.assertEquals(positionCount, output.getPositionCount());
+ Assert.assertTrue(output.mayHaveNull());
+ for (int i = 0; i < positionCount; i++) {
+ Assert.assertEquals(i % 2 == 0, output.isNull(i));
+ if (i % 2 != 0) {
+ Assert.assertEquals(values[i], output.getBinary(i));
+ }
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
new file mode 100644
index 0000000000..b4457445a4
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/ByteArrayColumnEncoderTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.common.block;
+
+import org.apache.iotdb.tsfile.read.common.block.column.BooleanColumn;
+import org.apache.iotdb.tsfile.read.common.block.column.BooleanColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Optional;
+import java.util.Random;
+
+public class ByteArrayColumnEncoderTest {
+ @Test
+ public void testBooleanColumn() {
+ final int positionCount = 10;
+
+ Random random = new Random();
+
+ boolean[] nullIndicators = new boolean[positionCount];
+ boolean[] values = new boolean[positionCount];
+ for (int i = 0; i < positionCount; i++) {
+ nullIndicators[i] = i % 2 == 0;
+ if (i % 2 != 0) {
+ values[i] = random.nextBoolean();
+ }
+ }
+ BooleanColumn input = new BooleanColumn(positionCount, Optional.of(nullIndicators), values);
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.BYTE_ARRAY);
+
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ try {
+ encoder.writeColumn(dos, input);
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+ BooleanColumnBuilder booleanColumnBuilder = new BooleanColumnBuilder(null, positionCount);
+ encoder.readColumn(booleanColumnBuilder, buffer, positionCount);
+ BooleanColumn output = (BooleanColumn) booleanColumnBuilder.build();
+ Assert.assertEquals(positionCount, output.getPositionCount());
+ Assert.assertTrue(output.mayHaveNull());
+ for (int i = 0; i < positionCount; i++) {
+ Assert.assertEquals(i % 2 == 0, output.isNull(i));
+ if (i % 2 != 0) {
+ Assert.assertEquals(values[i], output.getBoolean(i));
+ }
+ }
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
index 9feaf9bbb2..3b3dca7fb9 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int32ArrayColumnEncoderTest.java
@@ -19,9 +19,11 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
import org.apache.iotdb.tsfile.read.common.block.column.FloatColumn;
import org.apache.iotdb.tsfile.read.common.block.column.FloatColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.Int32ArrayColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.IntColumn;
import org.apache.iotdb.tsfile.read.common.block.column.IntColumnBuilder;
@@ -33,27 +35,30 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Optional;
+import java.util.Random;
public class Int32ArrayColumnEncoderTest {
@Test
public void testIntColumn() {
final int positionCount = 10;
+ Random random = new Random();
+
boolean[] nullIndicators = new boolean[positionCount];
int[] values = new int[positionCount];
for (int i = 0; i < positionCount; i++) {
nullIndicators[i] = i % 2 == 0;
if (i % 2 != 0) {
- values[i] = i;
+ values[i] = random.nextInt();
}
}
IntColumn input = new IntColumn(positionCount, Optional.of(nullIndicators), values);
- Int32ArrayColumnEncoder serde = new Int32ArrayColumnEncoder();
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
try {
- serde.writeColumn(dos, input);
+ encoder.writeColumn(dos, input);
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
@@ -61,14 +66,14 @@ public class Int32ArrayColumnEncoderTest {
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
IntColumnBuilder intColumnBuilder = new IntColumnBuilder(null, positionCount);
- serde.readColumn(intColumnBuilder, buffer, positionCount);
+ encoder.readColumn(intColumnBuilder, buffer, positionCount);
IntColumn output = (IntColumn) intColumnBuilder.build();
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
- Assert.assertEquals(i, output.getInt(i));
+ Assert.assertEquals(values[i], output.getInt(i));
}
}
}
@@ -77,21 +82,23 @@ public class Int32ArrayColumnEncoderTest {
public void testFloatColumn() {
final int positionCount = 10;
+ Random random = new Random();
+
boolean[] nullIndicators = new boolean[positionCount];
float[] values = new float[positionCount];
for (int i = 0; i < positionCount; i++) {
nullIndicators[i] = i % 2 == 0;
if (i % 2 != 0) {
- values[i] = i + i / 10F;
+ values[i] = random.nextFloat();
}
}
FloatColumn input = new FloatColumn(positionCount, Optional.of(nullIndicators), values);
- Int32ArrayColumnEncoder serde = new Int32ArrayColumnEncoder();
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
try {
- serde.writeColumn(dos, input);
+ encoder.writeColumn(dos, input);
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
@@ -99,14 +106,14 @@ public class Int32ArrayColumnEncoderTest {
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
FloatColumnBuilder floatColumnBuilder = new FloatColumnBuilder(null, positionCount);
- serde.readColumn(floatColumnBuilder, buffer, positionCount);
+ encoder.readColumn(floatColumnBuilder, buffer, positionCount);
FloatColumn output = (FloatColumn) floatColumnBuilder.build();
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
- Assert.assertEquals(i + i / 10F, output.getFloat(i), 0.001F);
+ Assert.assertEquals(values[i], output.getFloat(i), 0.001F);
}
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
index e094205dfb..048801f1da 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/Int64ArrayColumnEncoderTest.java
@@ -19,9 +19,11 @@
package org.apache.iotdb.tsfile.common.block;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoderFactory;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn;
import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.Int64ArrayColumnEncoder;
import org.apache.iotdb.tsfile.read.common.block.column.LongColumn;
import org.apache.iotdb.tsfile.read.common.block.column.LongColumnBuilder;
@@ -33,6 +35,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Optional;
+import java.util.Random;
public class Int64ArrayColumnEncoderTest {
@@ -40,21 +43,23 @@ public class Int64ArrayColumnEncoderTest {
public void testLongColumn() {
final int positionCount = 10;
+ Random random = new Random();
+
boolean[] nullIndicators = new boolean[positionCount];
long[] values = new long[positionCount];
for (int i = 0; i < positionCount; i++) {
nullIndicators[i] = i % 2 == 0;
if (i % 2 != 0) {
- values[i] = i;
+ values[i] = random.nextLong();
}
}
LongColumn input = new LongColumn(positionCount, Optional.of(nullIndicators), values);
- Int64ArrayColumnEncoder serde = new Int64ArrayColumnEncoder();
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.INT64_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
try {
- serde.writeColumn(dos, input);
+ encoder.writeColumn(dos, input);
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
@@ -62,14 +67,14 @@ public class Int64ArrayColumnEncoderTest {
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
LongColumnBuilder longColumnBuilder = new LongColumnBuilder(null, positionCount);
- serde.readColumn(longColumnBuilder, buffer, positionCount);
+ encoder.readColumn(longColumnBuilder, buffer, positionCount);
LongColumn output = (LongColumn) longColumnBuilder.build();
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
- Assert.assertEquals(i, output.getLong(i));
+ Assert.assertEquals(values[i], output.getLong(i));
}
}
}
@@ -78,21 +83,23 @@ public class Int64ArrayColumnEncoderTest {
public void testDoubleColumn() {
final int positionCount = 10;
+ Random random = new Random();
+
boolean[] nullIndicators = new boolean[positionCount];
double[] values = new double[positionCount];
for (int i = 0; i < positionCount; i++) {
nullIndicators[i] = i % 2 == 0;
if (i % 2 != 0) {
- values[i] = i + i / 10D;
+ values[i] = random.nextDouble();
}
}
DoubleColumn input = new DoubleColumn(positionCount, Optional.of(nullIndicators), values);
- Int64ArrayColumnEncoder serde = new Int64ArrayColumnEncoder();
+ ColumnEncoder encoder = ColumnEncoderFactory.get(ColumnEncoding.INT64_ARRAY);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
try {
- serde.writeColumn(dos, input);
+ encoder.writeColumn(dos, input);
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
@@ -100,14 +107,14 @@ public class Int64ArrayColumnEncoderTest {
ByteBuffer buffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
DoubleColumnBuilder doubleColumnBuilder = new DoubleColumnBuilder(null, positionCount);
- serde.readColumn(doubleColumnBuilder, buffer, positionCount);
+ encoder.readColumn(doubleColumnBuilder, buffer, positionCount);
DoubleColumn output = (DoubleColumn) doubleColumnBuilder.build();
Assert.assertEquals(positionCount, output.getPositionCount());
Assert.assertTrue(output.mayHaveNull());
for (int i = 0; i < positionCount; i++) {
Assert.assertEquals(i % 2 == 0, output.isNull(i));
if (i % 2 != 0) {
- Assert.assertEquals(i + i / 10D, output.getDouble(i), 0.001D);
+ Assert.assertEquals(values[i], output.getDouble(i), 0.001D);
}
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockSerdeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockSerdeTest.java
index 40c10c8358..eb69736dbf 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockSerdeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockSerdeTest.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnEncoding;
import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.Binary;
import org.junit.Assert;
import org.junit.Test;
@@ -45,18 +46,24 @@ public class TsBlockSerdeTest {
dataTypes.add(TSDataType.FLOAT);
dataTypes.add(TSDataType.INT64);
dataTypes.add(TSDataType.DOUBLE);
+ dataTypes.add(TSDataType.BOOLEAN);
+ dataTypes.add(TSDataType.TEXT);
TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(dataTypes);
ColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder intColumnBuilder = tsBlockBuilder.getColumnBuilder(0);
ColumnBuilder floatColumnBuilder = tsBlockBuilder.getColumnBuilder(1);
ColumnBuilder longColumnBuilder = tsBlockBuilder.getColumnBuilder(2);
ColumnBuilder doubleColumnBuilder = tsBlockBuilder.getColumnBuilder(3);
+ ColumnBuilder booleanColumnBuilder = tsBlockBuilder.getColumnBuilder(4);
+ ColumnBuilder binaryColumnBuilder = tsBlockBuilder.getColumnBuilder(5);
for (int i = 0; i < positionCount; i++) {
timeColumnBuilder.writeLong(i);
intColumnBuilder.writeInt(i);
floatColumnBuilder.writeFloat(i + i / 10F);
longColumnBuilder.writeLong(i);
doubleColumnBuilder.writeDouble(i + i / 10D);
+ booleanColumnBuilder.writeBoolean(true);
+ binaryColumnBuilder.writeBinary(new Binary("foo"));
tsBlockBuilder.declarePosition();
}
@@ -65,17 +72,21 @@ public class TsBlockSerdeTest {
ByteBuffer output = tsBlockSerde.serialize(tsBlockBuilder.build());
output.rewind();
int valueColumnCount = output.getInt();
- Assert.assertEquals(4, valueColumnCount);
+ Assert.assertEquals(6, valueColumnCount);
Assert.assertEquals(TSDataType.INT32, TSDataType.deserialize(output.get()));
Assert.assertEquals(TSDataType.FLOAT, TSDataType.deserialize(output.get()));
Assert.assertEquals(TSDataType.INT64, TSDataType.deserialize(output.get()));
Assert.assertEquals(TSDataType.DOUBLE, TSDataType.deserialize(output.get()));
+ Assert.assertEquals(TSDataType.BOOLEAN, TSDataType.deserialize(output.get()));
+ Assert.assertEquals(TSDataType.TEXT, TSDataType.deserialize(output.get()));
Assert.assertEquals(positionCount, output.getInt());
Assert.assertEquals(ColumnEncoding.INT64_ARRAY, ColumnEncoding.deserializeFrom(output));
Assert.assertEquals(ColumnEncoding.INT32_ARRAY, ColumnEncoding.deserializeFrom(output));
Assert.assertEquals(ColumnEncoding.INT32_ARRAY, ColumnEncoding.deserializeFrom(output));
Assert.assertEquals(ColumnEncoding.INT64_ARRAY, ColumnEncoding.deserializeFrom(output));
Assert.assertEquals(ColumnEncoding.INT64_ARRAY, ColumnEncoding.deserializeFrom(output));
+ Assert.assertEquals(ColumnEncoding.BYTE_ARRAY, ColumnEncoding.deserializeFrom(output));
+ Assert.assertEquals(ColumnEncoding.BINARY_ARRAY, ColumnEncoding.deserializeFrom(output));
output.rewind();
TsBlock tsBlock = tsBlockSerde.deserialize(output);
@@ -84,11 +95,15 @@ public class TsBlockSerdeTest {
Assert.assertEquals(TSDataType.FLOAT, tsBlock.getColumn(1).getDataType());
Assert.assertEquals(TSDataType.INT64, tsBlock.getColumn(2).getDataType());
Assert.assertEquals(TSDataType.DOUBLE, tsBlock.getColumn(3).getDataType());
+ Assert.assertEquals(TSDataType.BOOLEAN, tsBlock.getColumn(4).getDataType());
+ Assert.assertEquals(TSDataType.TEXT, tsBlock.getColumn(5).getDataType());
Assert.assertEquals(positionCount, tsBlock.getPositionCount());
Assert.assertEquals(ColumnEncoding.INT32_ARRAY, tsBlock.getColumn(0).getEncoding());
Assert.assertEquals(ColumnEncoding.INT32_ARRAY, tsBlock.getColumn(1).getEncoding());
Assert.assertEquals(ColumnEncoding.INT64_ARRAY, tsBlock.getColumn(2).getEncoding());
Assert.assertEquals(ColumnEncoding.INT64_ARRAY, tsBlock.getColumn(3).getEncoding());
+ Assert.assertEquals(ColumnEncoding.BYTE_ARRAY, tsBlock.getColumn(4).getEncoding());
+ Assert.assertEquals(ColumnEncoding.BINARY_ARRAY, tsBlock.getColumn(5).getEncoding());
} catch (IOException e) {
e.printStackTrace();
Assert.fail();