You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ji...@apache.org on 2026/06/03 07:17:48 UTC
(tsfile) branch develop updated: Add Tablet.serializedSize() and comprehensive size validation tests. (#824)
This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 86ec4b992 Add Tablet.serializedSize() and comprehensive size validation tests. (#824)
86ec4b992 is described below
commit 86ec4b992b8b14e73daed915b90c8da8528a9213
Author: Zhenyu Luo <zh...@apache.org>
AuthorDate: Wed Jun 3 15:17:43 2026 +0800
Add Tablet.serializedSize() and comprehensive size validation tests. (#824)
* Add Tablet.serializedSize() and comprehensive size validation tests.
Pre-allocate serialization buffer using exact size estimation, support OBJECT
type in tablet serialize/deserialize path, and consolidate serializedSize tests.
* Fix serialized size charset and overflow handling
* Avoid bitmap copy when sizing tablet serialization
---
.../org/apache/tsfile/utils/ReadWriteIOUtils.java | 12 +-
.../org/apache/tsfile/write/record/Tablet.java | 113 +++++-
.../tsfile/write/schema/MeasurementSchema.java | 12 +-
.../apache/tsfile/utils/ReadWriteIOUtilsTest.java | 7 +
.../org/apache/tsfile/write/record/TabletTest.java | 409 +++++++++++++++++++++
5 files changed, 540 insertions(+), 13 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
index 81b527529..59d2da32b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -185,7 +185,7 @@ public class ReadWriteIOUtils {
if (entry.getKey() == null) {
buffer.putInt(-1);
} else {
- bytes = entry.getKey().getBytes();
+ bytes = entry.getKey().getBytes(TSFileConfig.STRING_CHARSET);
buffer.putInt(bytes.length);
buffer.put(bytes);
length += bytes.length;
@@ -194,7 +194,7 @@ public class ReadWriteIOUtils {
if (entry.getValue() == null) {
buffer.putInt(-1);
} else {
- bytes = entry.getValue().getBytes();
+ bytes = entry.getValue().getBytes(TSFileConfig.STRING_CHARSET);
buffer.putInt(bytes.length);
buffer.put(bytes);
length += bytes.length;
@@ -509,7 +509,7 @@ public class ReadWriteIOUtils {
if (s == null) {
return INT_LEN;
}
- return INT_LEN + s.getBytes().length;
+ return INT_LEN + s.getBytes(TSFileConfig.STRING_CHARSET).length;
}
/** read a byte var from inputStream. */
@@ -1202,7 +1202,7 @@ public class ReadWriteIOUtils {
outputStream.write(NONE.ordinal());
} else {
outputStream.write(STRING.ordinal());
- byte[] bytes = value.toString().getBytes();
+ byte[] bytes = value.toString().getBytes(TSFileConfig.STRING_CHARSET);
outputStream.writeInt(bytes.length);
outputStream.write(bytes);
}
@@ -1238,7 +1238,7 @@ public class ReadWriteIOUtils {
byteBuffer.putInt(NONE.ordinal());
} else {
byteBuffer.putInt(STRING.ordinal());
- byte[] bytes = value.toString().getBytes();
+ byte[] bytes = value.toString().getBytes(TSFileConfig.STRING_CHARSET);
byteBuffer.putInt(bytes.length);
byteBuffer.put(bytes);
}
@@ -1271,7 +1271,7 @@ public class ReadWriteIOUtils {
length = buffer.getInt();
bytes = new byte[length];
buffer.get(bytes);
- return new String(bytes);
+ return new String(bytes, TSFileConfig.STRING_CHARSET);
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index 6093350e2..2bad6c953 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -748,13 +748,26 @@ public class Tablet implements Accountable {
/** Serialize {@link Tablet} */
public ByteBuffer serialize() throws IOException {
- try (PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final int serializedSize = serializedSize();
+ try (PublicBAOS byteArrayOutputStream = new PublicBAOS(serializedSize);
DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
serialize(outputStream);
return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
}
+ /** Return the exact serialized byte size of this tablet. */
+ public int serializedSize() {
+ int size = 0;
+ size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(insertTargetName));
+ size = Math.addExact(size, Integer.BYTES);
+ size = Math.addExact(size, serializedSizeOfMeasurementSchemas());
+ size = Math.addExact(size, serializedSizeOfTimes());
+ size = Math.addExact(size, serializedSizeOfBitMaps());
+ size = Math.addExact(size, serializedSizeOfValues());
+ return size;
+ }
+
public void serialize(DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(insertTargetName, stream);
ReadWriteIOUtils.write(rowSize, stream);
@@ -764,6 +777,104 @@ public class Tablet implements Accountable {
writeValues(stream);
}
+ private int serializedSizeOfMeasurementSchemas() {
+ int size = Byte.BYTES;
+ if (schemas != null) {
+ size = Math.addExact(size, Integer.BYTES);
+ for (int i = 0; i < schemas.size(); i++) {
+ size = Math.addExact(size, Byte.BYTES);
+ final IMeasurementSchema schema = schemas.get(i);
+ if (schema != null) {
+ size = Math.addExact(size, schema.serializedSize());
+ size = Math.addExact(size, Byte.BYTES);
+ }
+ }
+ }
+ return size;
+ }
+
+ private int serializedSizeOfTimes() {
+ int size = Byte.BYTES;
+ if (timestamps != null) {
+ size = Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize));
+ }
+ return size;
+ }
+
+ private int serializedSizeOfBitMaps() {
+ int size = Byte.BYTES;
+ if (bitMaps != null) {
+ final int columnCount = schemas == null ? 0 : schemas.size();
+ for (int i = 0; i < columnCount; i++) {
+ if (bitMaps[i] == null || bitMaps[i].isAllUnmarked(rowSize)) {
+ size = Math.addExact(size, Byte.BYTES);
+ } else {
+ size = Math.addExact(size, Byte.BYTES);
+ size = Math.addExact(size, Integer.BYTES);
+ size = Math.addExact(size, Integer.BYTES);
+ size = Math.addExact(size, BitMap.getSizeOfBytes(rowSize));
+ }
+ }
+ }
+ return size;
+ }
+
+ private int serializedSizeOfValues() {
+ int size = Byte.BYTES;
+ if (values != null) {
+ final int columnCount = schemas == null ? 0 : schemas.size();
+ for (int i = 0; i < columnCount; i++) {
+ size = Math.addExact(size, serializedSizeOfColumn(schemas.get(i).getType(), values[i]));
+ }
+ }
+ return size;
+ }
+
+ private int serializedSizeOfColumn(final TSDataType dataType, final Object column) {
+ int size = Byte.BYTES;
+ if (column == null) {
+ return size;
+ }
+ switch (dataType) {
+ case INT32:
+ return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize));
+ case DATE:
+ return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize));
+ case INT64:
+ case TIMESTAMP:
+ return Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize));
+ case FLOAT:
+ return Math.addExact(size, Math.multiplyExact(Float.BYTES, rowSize));
+ case DOUBLE:
+ return Math.addExact(size, Math.multiplyExact(Double.BYTES, rowSize));
+ case BOOLEAN:
+ return Math.addExact(size, rowSize);
+ case TEXT:
+ case STRING:
+ case BLOB:
+ case OBJECT:
+ return Math.addExact(size, serializedSizeOfBinaryValues((Binary[]) column));
+ default:
+ throw new UnSupportedDataTypeException(
+ Messages.format("error.write.type_not_supported", dataType));
+ }
+ }
+
+ private static int serializedSizeOfBinaryValues(final Binary[] binaryValues, final int rowSize) {
+ int size = 0;
+ for (int j = 0; j < rowSize; j++) {
+ size = Math.addExact(size, Byte.BYTES);
+ if (binaryValues[j] != null) {
+ size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[j]));
+ }
+ }
+ return size;
+ }
+
+ private int serializedSizeOfBinaryValues(final Binary[] binaryValues) {
+ return serializedSizeOfBinaryValues(binaryValues, rowSize);
+ }
+
/** Serialize {@link MeasurementSchema}s */
private void writeMeasurementSchemas(DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(BytesUtils.boolToByte(schemas != null), stream);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java
index aaaf7d841..16dab7789 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java
@@ -319,15 +319,15 @@ public class MeasurementSchema
@Override
public int serializedSize() {
int byteLen = 0;
- byteLen += ReadWriteIOUtils.sizeToWrite(measurementName);
- byteLen += 3 * Byte.BYTES;
+ byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(measurementName));
+ byteLen = Math.addExact(byteLen, 3 * Byte.BYTES);
if (props == null) {
- byteLen += Integer.BYTES;
+ byteLen = Math.addExact(byteLen, Integer.BYTES);
} else {
- byteLen += Integer.BYTES;
+ byteLen = Math.addExact(byteLen, Integer.BYTES);
for (Map.Entry<String, String> entry : props.entrySet()) {
- byteLen += ReadWriteIOUtils.sizeToWrite(entry.getKey());
- byteLen += ReadWriteIOUtils.sizeToWrite(entry.getValue());
+ byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(entry.getKey()));
+ byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(entry.getValue()));
}
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
index a0cb9a0a0..3b0b20a24 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
@@ -184,6 +184,13 @@ public class ReadWriteIOUtilsTest {
Assert.assertNotNull(result);
Assert.assertEquals(map, result);
+ ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE);
+ ReadWriteIOUtils.write(map, buffer);
+ buffer.flip();
+ result = ReadWriteIOUtils.readMap(buffer);
+ Assert.assertNotNull(result);
+ Assert.assertEquals(map, result);
+
// 7. null
map = null;
byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
index 65911c18a..ab4bf377b 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
@@ -22,26 +22,34 @@ package org.apache.tsfile.write.record;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
import org.junit.Test;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -147,6 +155,7 @@ public class TabletTest {
measurementSchemas.add(new MeasurementSchema("s7", TSDataType.BLOB, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s8", TSDataType.TIMESTAMP, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s9", TSDataType.DATE, TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s10", TSDataType.OBJECT, TSEncoding.PLAIN));
final int rowSize = 1000;
final Tablet tablet = new Tablet(deviceId, measurementSchemas);
@@ -170,6 +179,7 @@ public class TabletTest {
measurementSchemas.get(9).getMeasurementName(),
i,
LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1));
+ tablet.addValue(i, 10, i % 2 == 0, (long) i, new byte[] {(byte) i, (byte) (i + 1)});
tablet.getBitMaps()[i % measurementSchemas.size()].mark(i);
}
@@ -186,9 +196,11 @@ public class TabletTest {
tablet.addValue(measurementSchemas.get(7).getMeasurementName(), rowSize - 1, null);
tablet.addValue(measurementSchemas.get(8).getMeasurementName(), rowSize - 1, null);
tablet.addValue(measurementSchemas.get(9).getMeasurementName(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(10).getMeasurementName(), rowSize - 1, null);
try {
final ByteBuffer byteBuffer = tablet.serialize();
+ assertEquals(tablet.serializedSize(), byteBuffer.remaining());
final Tablet newTablet = Tablet.deserialize(byteBuffer);
assertEquals(tablet, newTablet);
for (int i = 0; i < rowSize; i++) {
@@ -357,6 +369,390 @@ public class TabletTest {
Assert.assertTrue(deserializeTablet.isNull(1, 0));
}
+ private static final Set<TSDataType> NON_SERIALIZABLE_DATA_TYPES =
+ EnumSet.of(TSDataType.VECTOR, TSDataType.UNKNOWN);
+
+ private static final List<TSDataType> SERIALIZABLE_DATA_TYPES =
+ Arrays.stream(TSDataType.values())
+ .filter(dataType -> !NON_SERIALIZABLE_DATA_TYPES.contains(dataType))
+ .collect(Collectors.toList());
+
+ private static final int[] ROW_COUNTS_FOR_SIZE_TEST = {0, 1, 7, 50};
+
+ @Test
+ public void testSerializedSizeMatchesActualSize() throws IOException {
+ // tree model: single column per type
+ for (final TSDataType type : SERIALIZABLE_DATA_TYPES) {
+ for (final int rowCount : ROW_COUNTS_FOR_SIZE_TEST) {
+ assertSerializedSizeMatches(
+ createAndFillTreeTablet(
+ "root.sg.d1",
+ columnNamesForType(type),
+ Arrays.asList(type),
+ rowCount,
+ 0,
+ false,
+ false),
+ "tree single column " + type + " rows=" + rowCount);
+ }
+ }
+
+ // table model: single column per type
+ for (final TSDataType type : SERIALIZABLE_DATA_TYPES) {
+ for (final int rowCount : ROW_COUNTS_FOR_SIZE_TEST) {
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "table1",
+ columnNamesForType(type),
+ Arrays.asList(type),
+ ColumnCategory.nCopy(ColumnCategory.FIELD, 1),
+ rowCount,
+ 0,
+ false,
+ false),
+ "table single column " + type + " rows=" + rowCount);
+ }
+ }
+
+ // all types combined
+ final List<TSDataType> treeTypes = SERIALIZABLE_DATA_TYPES;
+ final List<TSDataType> tableTypes = new ArrayList<>();
+ tableTypes.add(TSDataType.STRING);
+ tableTypes.addAll(treeTypes);
+ for (final int rowCount : new int[] {1, 25, 100}) {
+ assertSerializedSizeMatches(
+ createAndFillTreeTablet(
+ "root.sg.d1", buildColumnNames(treeTypes), treeTypes, rowCount, 100, false, false),
+ "tree all types combined rows=" + rowCount);
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "table1",
+ buildColumnNames(tableTypes),
+ tableTypes,
+ buildTableColumnCategories(tableTypes.size()),
+ rowCount,
+ 100,
+ false,
+ false),
+ "table all types combined rows=" + rowCount);
+ }
+
+ // variable-length binary columns
+ final List<TSDataType> binaryTypes =
+ Arrays.asList(TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT);
+ assertSerializedSizeMatches(
+ createAndFillTreeTablet(
+ "root.sg.d1", buildColumnNames(binaryTypes), binaryTypes, 30, 0, false, true),
+ "tree variable binary lengths");
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "table1",
+ buildColumnNames(binaryTypes),
+ binaryTypes,
+ ColumnCategory.nCopy(ColumnCategory.FIELD, binaryTypes.size()),
+ 30,
+ 0,
+ false,
+ true),
+ "table variable binary lengths");
+
+ // sparse null values
+ assertSerializedSizeMatches(
+ createAndFillTreeTablet(
+ "root.sg.d1", buildColumnNames(treeTypes), treeTypes, 40, 0, true, false),
+ "tree with null values");
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "table1",
+ buildColumnNames(tableTypes),
+ tableTypes,
+ buildTableColumnCategories(tableTypes.size()),
+ 40,
+ 0,
+ true,
+ false),
+ "table with null values");
+
+ // table model with TAG columns
+ final List<String> tagColumnNames = new ArrayList<>();
+ final List<TSDataType> tagDataTypes = new ArrayList<>();
+ final List<ColumnCategory> tagCategories = new ArrayList<>();
+ tagColumnNames.add("region");
+ tagDataTypes.add(TSDataType.STRING);
+ tagCategories.add(ColumnCategory.TAG);
+ for (int i = 0; i < SERIALIZABLE_DATA_TYPES.size(); i++) {
+ tagColumnNames.add("m" + i);
+ tagDataTypes.add(SERIALIZABLE_DATA_TYPES.get(i));
+ tagCategories.add(ColumnCategory.FIELD);
+ }
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "metrics_table", tagColumnNames, tagDataTypes, tagCategories, 20, 0, false, true),
+ "table model with TAG columns");
+
+ // mixed fixed-length and variable-length columns
+ final List<TSDataType> mixedTypes =
+ Arrays.asList(
+ TSDataType.INT32,
+ TSDataType.TEXT,
+ TSDataType.STRING,
+ TSDataType.BLOB,
+ TSDataType.DOUBLE);
+ assertSerializedSizeMatches(
+ createAndFillTreeTablet(
+ "root.sg.d1", buildColumnNames(mixedTypes), mixedTypes, 15, 5, false, true),
+ "tree mixed column payload lengths");
+ assertSerializedSizeMatches(
+ createAndFillTableTablet(
+ "table1",
+ buildColumnNames(mixedTypes),
+ mixedTypes,
+ ColumnCategory.nCopy(ColumnCategory.FIELD, mixedTypes.size()),
+ 15,
+ 5,
+ false,
+ true),
+ "table mixed column payload lengths");
+
+ // OBJECT column via dedicated write API
+ final List<IMeasurementSchema> objectSchemas =
+ Arrays.asList(new MeasurementSchema("obj", TSDataType.OBJECT, TSEncoding.PLAIN));
+ final Tablet objectTablet = new Tablet("root.sg.d1", objectSchemas, 5);
+ for (int i = 0; i < 5; i++) {
+ objectTablet.addTimestamp(i, i);
+ objectTablet.addValue(i, 0, i % 2 == 0, i * 10L, new byte[] {(byte) i, (byte) (i + 1)});
+ }
+ assertSerializedSizeMatches(objectTablet, "tree OBJECT column");
+ final Tablet deserializedObject = Tablet.deserialize(objectTablet.serialize());
+ assertEquals(objectTablet, deserializedObject);
+ for (int i = 0; i < 5; i++) {
+ assertEquals(objectTablet.getValue(i, 0), deserializedObject.getValue(i, 0));
+ }
+
+ final Map<String, String> propsWithNonAscii = new HashMap<>();
+ propsWithNonAscii.put("编码", "字典");
+ final Tablet nonAsciiTreeTablet =
+ new Tablet(
+ "root.测试.设备1",
+ Arrays.asList(
+ new MeasurementSchema(
+ "温度",
+ TSDataType.TEXT,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED,
+ propsWithNonAscii)),
+ 3);
+ for (int i = 0; i < 3; i++) {
+ nonAsciiTreeTablet.addTimestamp(i, i);
+ nonAsciiTreeTablet.addValue("温度", i, "值" + i);
+ }
+ assertSerializedSizeMatches(nonAsciiTreeTablet, "tree non-ASCII names and schema props");
+
+ final Tablet nonAsciiTableTablet =
+ createAndFillTableTablet(
+ "表一",
+ Arrays.asList("标签", "数值"),
+ Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE),
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD),
+ 3,
+ 0,
+ false,
+ true);
+ assertSerializedSizeMatches(nonAsciiTableTablet, "table non-ASCII names");
+ }
+
+ private static List<ColumnCategory> buildTableColumnCategories(int columnCount) {
+ final List<ColumnCategory> categories = new ArrayList<>(columnCount);
+ categories.add(ColumnCategory.TAG);
+ for (int i = 1; i < columnCount; i++) {
+ categories.add(ColumnCategory.FIELD);
+ }
+ return categories;
+ }
+
+ private static List<String> buildColumnNames(List<TSDataType> dataTypes) {
+ final List<String> names = new ArrayList<>(dataTypes.size());
+ for (int i = 0; i < dataTypes.size(); i++) {
+ if (i == 0 && dataTypes.size() > 1) {
+ names.add("tag");
+ } else {
+ names.add("m_" + dataTypes.get(i).name() + "_" + i);
+ }
+ }
+ return names;
+ }
+
+ private static List<String> columnNamesForType(TSDataType type) {
+ return Arrays.asList("m_" + type.name() + "_0");
+ }
+
+ private Tablet createAndFillTreeTablet(
+ String deviceId,
+ List<String> columnNames,
+ List<TSDataType> dataTypes,
+ int rowCount,
+ int valueOffset,
+ boolean withNulls,
+ boolean variableBinaryLength)
+ throws IOException {
+ validateTabletSchema(columnNames, dataTypes, null);
+ final List<IMeasurementSchema> schemas = new ArrayList<>(dataTypes.size());
+ for (int i = 0; i < dataTypes.size(); i++) {
+ schemas.add(new MeasurementSchema(columnNames.get(i), dataTypes.get(i), TSEncoding.PLAIN));
+ }
+ final Tablet tablet = new Tablet(deviceId, schemas, Math.max(1024, rowCount + 1));
+ fillTabletRows(tablet, rowCount, valueOffset, withNulls, variableBinaryLength);
+ return tablet;
+ }
+
+ private Tablet createAndFillTableTablet(
+ String tableName,
+ List<String> columnNames,
+ List<TSDataType> dataTypes,
+ List<ColumnCategory> columnCategories,
+ int rowCount,
+ int valueOffset,
+ boolean withNulls,
+ boolean variableBinaryLength)
+ throws IOException {
+ validateTabletSchema(columnNames, dataTypes, columnCategories);
+ final Tablet tablet =
+ new Tablet(
+ tableName, columnNames, dataTypes, columnCategories, Math.max(1024, rowCount + 1));
+ fillTabletRows(tablet, rowCount, valueOffset, withNulls, variableBinaryLength);
+ return tablet;
+ }
+
+ private static void validateTabletSchema(
+ List<String> columnNames, List<TSDataType> dataTypes, List<ColumnCategory> columnCategories) {
+ if (columnNames.size() != dataTypes.size()) {
+ throw new IllegalArgumentException(
+ "columnNames size "
+ + columnNames.size()
+ + " must match dataTypes size "
+ + dataTypes.size());
+ }
+ if (columnCategories != null && columnCategories.size() != dataTypes.size()) {
+ throw new IllegalArgumentException(
+ "columnCategories size "
+ + columnCategories.size()
+ + " must match dataTypes size "
+ + dataTypes.size());
+ }
+ }
+
+ private void fillTabletRows(
+ Tablet tablet,
+ int rowCount,
+ int valueOffset,
+ boolean withNulls,
+ boolean variableBinaryLength) {
+ if (rowCount > 0) {
+ fillTabletForSerializedSizeTest(
+ tablet, valueOffset, rowCount, withNulls, variableBinaryLength);
+ }
+ }
+
+ private void fillTabletForSerializedSizeTest(
+ Tablet tablet,
+ int valueOffset,
+ int rowCount,
+ boolean withNulls,
+ boolean variableBinaryLength) {
+ for (int row = 0; row < rowCount; row++) {
+ tablet.addTimestamp(row, valueOffset + row);
+ for (int col = 0; col < tablet.getSchemas().size(); col++) {
+ final TSDataType type = tablet.getSchemas().get(col).getType();
+ if (isNullCell(withNulls, row, col)) {
+ tablet.addValue(tablet.getSchemas().get(col).getMeasurementName(), row, null);
+ } else if (type == TSDataType.OBJECT) {
+ tablet.addValue(
+ row,
+ col,
+ (row + col) % 2 == 0,
+ valueOffset + row * 1000L + col,
+ payloadBytes(binaryPayloadLength(variableBinaryLength, row, col)));
+ } else {
+ tablet.addValue(
+ tablet.getSchemas().get(col).getMeasurementName(),
+ row,
+ sampleValue(type, row, col, variableBinaryLength));
+ }
+ }
+ }
+ }
+
+ private static boolean isNullCell(boolean withNulls, int row, int col) {
+ return withNulls && (row + col) % 3 == 0;
+ }
+
+ private static int binaryPayloadLength(boolean variableBinaryLength, int row, int col) {
+ if (variableBinaryLength) {
+ return (col + 1) * 17 + row * 3 + 1;
+ }
+ return 8 + row % 11;
+ }
+
+ private Object sampleValue(TSDataType type, int row, int col, boolean variableBinaryLength) {
+ switch (type) {
+ case BOOLEAN:
+ return (row + col) % 2 == 0;
+ case INT32:
+ return row + col * 100;
+ case INT64:
+ case TIMESTAMP:
+ return (long) (valueOffset(row, col) * 1_000_000L);
+ case FLOAT:
+ return (row + col) * 1.5f;
+ case DOUBLE:
+ return (row + col) * 2.5;
+ case TEXT:
+ case STRING:
+ return stringOfLength(binaryPayloadLength(variableBinaryLength, row, col));
+ case BLOB:
+ return binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col));
+ case DATE:
+ return LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1);
+ default:
+ throw new IllegalArgumentException("Unsupported type in test: " + type);
+ }
+ }
+
+ private static int valueOffset(int row, int col) {
+ return row + col + 1;
+ }
+
+ private static String stringOfLength(int length) {
+ final char[] chars = new char[length];
+ Arrays.fill(chars, 'x');
+ return new String(chars);
+ }
+
+ private static Binary binaryOfLength(int length) {
+ final byte[] bytes = new byte[length];
+ Arrays.fill(bytes, (byte) 'b');
+ return new Binary(bytes);
+ }
+
+ private static byte[] payloadBytes(int length) {
+ final byte[] bytes = new byte[length];
+ Arrays.fill(bytes, (byte) 'p');
+ return bytes;
+ }
+
+ private void assertSerializedSizeMatches(Tablet tablet, String scenario) throws IOException {
+ final int expectedSize = tablet.serializedSize();
+ final ByteBuffer buffer = tablet.serialize();
+ assertEquals(scenario + ": serialize() buffer size", expectedSize, buffer.remaining());
+ try (PublicBAOS baos = new PublicBAOS();
+ DataOutputStream outputStream = new DataOutputStream(baos)) {
+ tablet.serialize(outputStream);
+ assertEquals(scenario + ": serialize(stream) size", expectedSize, baos.size());
+ }
+ buffer.rewind();
+ assertEquals(scenario + ": deserialize roundtrip", tablet, Tablet.deserialize(buffer));
+ }
+
@Test
public void testAppendInconsistent() {
Tablet t1 =
@@ -425,6 +821,9 @@ public class TabletTest {
case BLOB:
t.addValue(i, j, String.valueOf(i + valueOffset));
break;
+ case OBJECT:
+ t.addValue(i, j, (i + valueOffset) % 2 == 0, i + valueOffset, new byte[] {(byte) i});
+ break;
case DATE:
t.addValue(i, j, LocalDate.of(i + valueOffset, 1, 1));
break;
@@ -655,6 +1054,16 @@ public class TabletTest {
new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)),
result.getValue(i, j));
break;
+ case OBJECT:
+ {
+ byte[] content = new byte[] {(byte) i};
+ byte[] expected = new byte[content.length + 9];
+ expected[0] = (byte) (i % 2);
+ System.arraycopy(BytesUtils.longToBytes(i), 0, expected, 1, 8);
+ System.arraycopy(content, 0, expected, 9, content.length);
+ assertEquals(new Binary(expected), result.getValue(i, j));
+ }
+ break;
case DATE:
assertEquals(LocalDate.of(i, 1, 1), result.getValue(i, j));
break;