You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ha...@apache.org on 2024/09/05 10:49:38 UTC
(tsfile) branch develop updated: Fixed the bug that local date may put null value to tablet (#234)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 5dc645c2 Fixed the bug that local date may put null value to tablet (#234)
5dc645c2 is described below
commit 5dc645c2ebd77713f3b1ba7c036854c9eb33b5e5
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Thu Sep 5 18:49:33 2024 +0800
Fixed the bug that local date may put null value to tablet (#234)
---
.../org/apache/tsfile/write/record/Tablet.java | 23 ++++----
.../org/apache/tsfile/write/record/TabletTest.java | 64 ++++++++++++++++------
2 files changed, 59 insertions(+), 28 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index 7b5b9910..e0213197 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -57,6 +57,7 @@ public class Tablet {
private static final int DEFAULT_SIZE = 1024;
private static final String NOT_SUPPORT_DATATYPE = "Data type %s is not supported.";
+ private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1);
/** DeviceId if using tree-view interfaces or TableName when using table-view interfaces. */
private String insertTargetName;
@@ -220,14 +221,14 @@ public class Tablet {
timestamps[rowIndex] = timestamp;
}
- public void addValue(String measurementId, int rowIndex, Object value) {
+ public void addValue(final String measurementId, final int rowIndex, final Object value) {
int indexOfSchema = measurementIndex.get(measurementId);
IMeasurementSchema measurementSchema = schemas.get(indexOfSchema);
addValueOfDataType(measurementSchema.getType(), rowIndex, indexOfSchema, value);
}
private void addValueOfDataType(
- TSDataType dataType, int rowIndex, int indexOfSchema, Object value) {
+ final TSDataType dataType, final int rowIndex, final int indexOfSchema, final Object value) {
if (value == null) {
// Init the bitMap to mark null value
@@ -245,9 +246,9 @@ public class Tablet {
case STRING:
case BLOB:
{
- Binary[] sensor = (Binary[]) values[indexOfSchema];
+ final Binary[] sensor = (Binary[]) values[indexOfSchema];
if (value instanceof Binary) {
- sensor[rowIndex] = value != null ? (Binary) value : Binary.EMPTY_VALUE;
+ sensor[rowIndex] = (Binary) value;
} else {
sensor[rowIndex] =
value != null
@@ -258,38 +259,38 @@ public class Tablet {
}
case FLOAT:
{
- float[] sensor = (float[]) values[indexOfSchema];
+ final float[] sensor = (float[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE;
break;
}
case INT32:
{
- int[] sensor = (int[]) values[indexOfSchema];
+ final int[] sensor = (int[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE;
break;
}
case DATE:
{
- LocalDate[] sensor = (LocalDate[]) values[indexOfSchema];
- sensor[rowIndex] = (LocalDate) value;
+ final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema];
+ sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE;
break;
}
case INT64:
case TIMESTAMP:
{
- long[] sensor = (long[]) values[indexOfSchema];
+ final long[] sensor = (long[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE;
break;
}
case DOUBLE:
{
- double[] sensor = (double[]) values[indexOfSchema];
+ final double[] sensor = (double[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE;
break;
}
case BOOLEAN:
{
- boolean[] sensor = (boolean[]) values[indexOfSchema];
+ final boolean[] sensor = (boolean[]) values[indexOfSchema];
sensor[rowIndex] = value != null && (boolean) value;
break;
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
index dd9f38b1..b5dcd3b0 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
@@ -19,8 +19,10 @@
package org.apache.tsfile.write.record;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -28,6 +30,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Test;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
@@ -37,14 +40,14 @@ import static org.junit.Assert.fail;
public class TabletTest {
@Test
public void testSerializationAndDeSerialization() {
- String deviceId = "root.sg";
- List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ final String deviceId = "root.sg";
+ final List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN));
- int rowSize = 100;
- long[] timestamps = new long[rowSize];
- Object[] values = new Object[2];
+ final int rowSize = 100;
+ final long[] timestamps = new long[rowSize];
+ final Object[] values = new Object[2];
values[0] = new int[rowSize];
values[1] = new long[rowSize];
@@ -54,7 +57,7 @@ public class TabletTest {
((long[]) values[1])[i] = 1;
}
- Tablet tablet =
+ final Tablet tablet =
new Tablet(
deviceId,
measurementSchemas,
@@ -63,10 +66,10 @@ public class TabletTest {
new BitMap[] {new BitMap(1024), new BitMap(1024)},
rowSize);
try {
- ByteBuffer byteBuffer = tablet.serialize();
- Tablet newTablet = Tablet.deserialize(byteBuffer);
+ final ByteBuffer byteBuffer = tablet.serialize();
+ final Tablet newTablet = Tablet.deserialize(byteBuffer);
assertEquals(newTablet, tablet);
- } catch (Exception e) {
+ } catch (final Exception e) {
e.printStackTrace();
fail();
}
@@ -74,20 +77,24 @@ public class TabletTest {
@Test
public void testSerializationAndDeSerializationWithMoreData() {
- String deviceId = "root.sg";
- List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ final String deviceId = "root.sg";
+ final List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s3", TSDataType.DOUBLE, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s5", TSDataType.TEXT, TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s6", TSDataType.STRING, TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s7", TSDataType.BLOB, TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s8", TSDataType.TIMESTAMP, TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s9", TSDataType.DATE, TSEncoding.PLAIN));
- int rowSize = 1000;
- Tablet tablet = new Tablet(deviceId, measurementSchemas);
+ final int rowSize = 1000;
+ final Tablet tablet = new Tablet(deviceId, measurementSchemas);
tablet.rowSize = rowSize;
tablet.initBitMaps();
- for (int i = 0; i < rowSize; i++) {
+ for (int i = 0; i < rowSize - 1; i++) {
tablet.addTimestamp(i, i);
tablet.addValue(measurementSchemas.get(0).getMeasurementId(), i, i);
tablet.addValue(measurementSchemas.get(1).getMeasurementId(), i, (long) i);
@@ -95,15 +102,38 @@ public class TabletTest {
tablet.addValue(measurementSchemas.get(3).getMeasurementId(), i, (double) i);
tablet.addValue(measurementSchemas.get(4).getMeasurementId(), i, (i % 2) == 0);
tablet.addValue(measurementSchemas.get(5).getMeasurementId(), i, String.valueOf(i));
+ tablet.addValue(measurementSchemas.get(6).getMeasurementId(), i, String.valueOf(i));
+ tablet.addValue(
+ measurementSchemas.get(7).getMeasurementId(),
+ i,
+ new Binary(String.valueOf(i), TSFileConfig.STRING_CHARSET));
+ tablet.addValue(measurementSchemas.get(8).getMeasurementId(), i, (long) i);
+ tablet.addValue(
+ measurementSchemas.get(9).getMeasurementId(),
+ i,
+ LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1));
tablet.bitMaps[i % measurementSchemas.size()].mark(i);
}
+ // Test add null
+ tablet.addTimestamp(rowSize - 1, rowSize - 1);
+ tablet.addValue(measurementSchemas.get(0).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(1).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(2).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(3).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(4).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(5).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(6).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(7).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(8).getMeasurementId(), rowSize - 1, null);
+ tablet.addValue(measurementSchemas.get(9).getMeasurementId(), rowSize - 1, null);
+
try {
- ByteBuffer byteBuffer = tablet.serialize();
- Tablet newTablet = Tablet.deserialize(byteBuffer);
+ final ByteBuffer byteBuffer = tablet.serialize();
+ final Tablet newTablet = Tablet.deserialize(byteBuffer);
assertEquals(newTablet, tablet);
- } catch (Exception e) {
+ } catch (final Exception e) {
e.printStackTrace();
fail();
}