You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/03/28 08:10:45 UTC
[iotdb] branch ty-mpp updated: Implement TsBlock
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty-mpp
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty-mpp by this push:
new 4795d2b6 Implement TsBlock
4795d2b6 is described below
commit 4795d2b6433b6a200fd657d0839dcc88ba2ea214
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Mar 28 16:06:30 2022 +0800
Implement TsBlock
---
pom.xml | 5 +
.../db/mpp/operator/process/TimeJoinOperator.java | 1 +
.../db/mpp/operator/source/SeriesScanUtil.java | 8 +-
.../query/reader/chunk/MemAlignedPageReader.java | 1 +
.../iotdb/db/query/reader/chunk/MemPageReader.java | 14 +-
tsfile/pom.xml | 4 +
.../read/common/block/column/BinaryColumn.java | 2 +-
.../common/block/column/BinaryColumnBuilder.java | 4 +-
.../read/common/block/column/BooleanColumn.java | 2 +-
.../common/block/column/BooleanColumnBuilder.java | 4 +-
.../read/common/block/column/DoubleColumn.java | 2 +-
.../common/block/column/DoubleColumnBuilder.java | 4 +-
.../read/common/block/column/FloatColumn.java | 2 +-
.../common/block/column/FloatColumnBuilder.java | 4 +-
.../tsfile/read/common/block/column/IntColumn.java | 2 +-
.../read/common/block/column/IntColumnBuilder.java | 4 +-
.../read/common/block/column/LongColumn.java | 2 +-
.../common/block/column/LongColumnBuilder.java | 4 +-
.../read/common/block/column/TimeColumn.java | 2 +-
.../common/block/column/TimeColumnBuilder.java | 2 +-
.../tsfile/read/reader/page/AlignedPageReader.java | 7 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 14 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 317 +++++++++++++++++++++
23 files changed, 380 insertions(+), 31 deletions(-)
diff --git a/pom.xml b/pom.xml
index f19c018..b87598a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -506,6 +506,11 @@
<version>${airline.version}</version>
</dependency>
<dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>slice</artifactId>
+ <version>0.41</version>
+ </dependency>
+ <dependency>
<groupId>org.openjdk.jol</groupId>
<artifactId>jol-core</artifactId>
<version>0.2</version>
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/operator/process/TimeJoinOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/operator/process/TimeJoinOperator.java
index 60fa4f2..e2537b7 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/operator/process/TimeJoinOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/operator/process/TimeJoinOperator.java
@@ -127,6 +127,7 @@ public class TimeJoinOperator implements ProcessOperator {
TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
while (!timeSelector.isEmpty() && timeSelector.first() <= currentEndTime) {
timeBuilder.writeLong(timeSelector.pollFirst());
+ tsBlockBuilder.declarePosition();
}
tsBlockBuilder.buildValueColumnBuilders(dataTypes);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/operator/source/SeriesScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/operator/source/SeriesScanUtil.java
index e3d2f9e..b369665 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/operator/source/SeriesScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/operator/source/SeriesScanUtil.java
@@ -757,7 +757,6 @@ public class SeriesScanUtil {
if (valueFilter == null
|| valueFilter.satisfy(timeValuePair.getTimestamp(), valueForFilter)) {
- builder.declarePosition();
timeBuilder.writeLong(timeValuePair.getTimestamp());
switch (dataType) {
case BOOLEAN:
@@ -781,12 +780,17 @@ public class SeriesScanUtil {
case VECTOR:
TsPrimitiveType[] values = timeValuePair.getValue().getVector();
for (int i = 0; i < values.length; i++) {
- builder.getColumnBuilder(i).writeTsPrimitiveType(values[i]);
+ if (values[i] == null) {
+ builder.getColumnBuilder(i).appendNull();
+ } else {
+ builder.getColumnBuilder(i).writeTsPrimitiveType(values[i]);
+ }
}
break;
default:
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
+ builder.declarePosition();
}
}
hasCachedNextOverlappedPage = builder.isEmpty();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemAlignedPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemAlignedPageReader.java
index e7ebc1c..7d679d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemAlignedPageReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemAlignedPageReader.java
@@ -112,6 +112,7 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader {
for (int i = 0; i < values.length; i++) {
builder.getColumnBuilder(i).writeTsPrimitiveType(values[i]);
}
+ builder.declarePosition();
}
}
return builder.build();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
index 16704a2..f8e87d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
@@ -70,9 +70,9 @@ public class MemPageReader implements IPageReader {
TSDataType dataType = chunkMetadata.getDataType();
// TODO we still need to consider data type, ascending and descending here
- TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Collections.singletonList(dataType));
- TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = tsBlockBuilder.getColumnBuilder(0);
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(dataType));
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
switch (dataType) {
case BOOLEAN:
while (timeValuePairIterator.hasNextTimeValuePair()) {
@@ -82,6 +82,7 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeBoolean(timeValuePair.getValue().getBoolean());
+ builder.declarePosition();
}
}
break;
@@ -93,6 +94,7 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeInt(timeValuePair.getValue().getInt());
+ builder.declarePosition();
}
}
break;
@@ -104,6 +106,7 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeLong(timeValuePair.getValue().getLong());
+ builder.declarePosition();
}
}
break;
@@ -115,6 +118,7 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeFloat(timeValuePair.getValue().getFloat());
+ builder.declarePosition();
}
}
break;
@@ -126,6 +130,7 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeDouble(timeValuePair.getValue().getDouble());
+ builder.declarePosition();
}
}
break;
@@ -137,13 +142,14 @@ public class MemPageReader implements IPageReader {
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
timeBuilder.writeLong(timeValuePair.getTimestamp());
valueBuilder.writeBinary(timeValuePair.getValue().getBinary());
+ builder.declarePosition();
}
}
break;
default:
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
- return tsBlockBuilder.build();
+ return builder.build();
}
@Override
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index 147ed65..44f949b 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -72,6 +72,10 @@
<groupId>org.openjdk.jol</groupId>
<artifactId>jol-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>slice</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java
index cfb22c9..8828393 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java
@@ -25,8 +25,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class BinaryColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
index c5c629f..ca95163 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
@@ -25,9 +25,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class BinaryColumnBuilder implements ColumnBuilder {
@@ -82,7 +82,7 @@ public class BinaryColumnBuilder implements ColumnBuilder {
int index = offset;
BinaryColumn column = (BinaryColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeBinary(column.getBinary(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java
index a2b8550..54544d3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java
@@ -24,8 +24,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class BooleanColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java
index 5e7629d..da5760e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java
@@ -24,9 +24,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class BooleanColumnBuilder implements ColumnBuilder {
@@ -84,7 +84,7 @@ public class BooleanColumnBuilder implements ColumnBuilder {
int index = offset;
BooleanColumn column = (BooleanColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeBoolean(column.getBoolean(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java
index d16e0c0..32809b0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java
@@ -24,8 +24,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class DoubleColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java
index a01f6f2..cdfcc35 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java
@@ -24,9 +24,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class DoubleColumnBuilder implements ColumnBuilder {
@@ -84,7 +84,7 @@ public class DoubleColumnBuilder implements ColumnBuilder {
int index = offset;
DoubleColumn column = (DoubleColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeDouble(column.getDouble(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java
index 6543bde..51a2675 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java
@@ -24,8 +24,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class FloatColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java
index ff89ea2..142e711 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java
@@ -24,9 +24,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class FloatColumnBuilder implements ColumnBuilder {
@@ -84,7 +84,7 @@ public class FloatColumnBuilder implements ColumnBuilder {
int index = offset;
FloatColumn column = (FloatColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeFloat(column.getFloat(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java
index 4ad843f..0d48dd2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java
@@ -24,8 +24,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class IntColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java
index 9ab3f7f..b9f8e67 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java
@@ -24,9 +24,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class IntColumnBuilder implements ColumnBuilder {
@@ -84,7 +84,7 @@ public class IntColumnBuilder implements ColumnBuilder {
int index = offset;
IntColumn column = (IntColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeInt(column.getInt(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java
index 9ab9edd..345e71d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java
@@ -24,8 +24,8 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Optional;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class LongColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java
index 9053913..12443fe 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java
@@ -24,9 +24,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class LongColumnBuilder implements ColumnBuilder {
@@ -84,7 +84,7 @@ public class LongColumnBuilder implements ColumnBuilder {
int index = offset;
LongColumn column = (LongColumn) valueColumn;
for (int i = 0; i < count; i++) {
- if (timeColumn.getLong(index) == timeBuilder.getTime(i)) {
+ if (timeColumn.getLong(index) == timeBuilder.getTime(i) && !valueColumn.isNull(index)) {
writeLong(column.getLong(index++));
} else {
appendNull();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java
index c8d4748..c46fbd2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java
@@ -20,8 +20,8 @@ package org.apache.iotdb.tsfile.read.common.block.column;
import org.openjdk.jol.info.ClassLayout;
+import static io.airlift.slice.SizeOf.sizeOf;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class TimeColumn implements Column {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java
index 9b455cb..89267c5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java
@@ -22,9 +22,9 @@ import org.openjdk.jol.info.ClassLayout;
import java.util.Arrays;
+import static io.airlift.slice.SizeOf.sizeOf;
import static java.lang.Math.max;
import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
-import static org.openjdk.jol.util.VMSupport.sizeOf;
public class TimeColumnBuilder implements ColumnBuilder {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index dd75b21..df8eaf8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -134,8 +134,13 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
if (!isNull && (filter == null || filter.satisfy(timestamp, notNullObject))) {
builder.getTimeColumnBuilder().writeLong(timestamp);
for (int i = 0; i < v.length; i++) {
- builder.getColumnBuilder(i).writeTsPrimitiveType(v[i]);
+ if (v[i] != null) {
+ builder.getColumnBuilder(i).writeTsPrimitiveType(v[i]);
+ } else {
+ builder.getColumnBuilder(i).appendNull();
+ }
}
+ builder.declarePosition();
}
}
return builder.build();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index ec5ff85..f7ce9bd 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -161,9 +161,9 @@ public class PageReader implements IPageReader {
@Override
public TsBlock getAllSatisfiedData(boolean ascending) throws IOException {
// TODO we still need to consider data type, ascending and descending here
- TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Collections.singletonList(dataType));
- TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
- ColumnBuilder valueBuilder = tsBlockBuilder.getColumnBuilder(0);
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(dataType));
+ TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder();
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(0);
if (filter == null || filter.satisfy(getStatistics())) {
switch (dataType) {
case BOOLEAN:
@@ -173,6 +173,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aBoolean))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeBoolean(aBoolean);
+ builder.declarePosition();
}
}
break;
@@ -183,6 +184,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, anInt))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeInt(anInt);
+ builder.declarePosition();
}
}
break;
@@ -193,6 +195,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aLong))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeLong(aLong);
+ builder.declarePosition();
}
}
break;
@@ -203,6 +206,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aFloat))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeFloat(aFloat);
+ builder.declarePosition();
}
}
break;
@@ -213,6 +217,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aDouble))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeDouble(aDouble);
+ builder.declarePosition();
}
}
break;
@@ -223,6 +228,7 @@ public class PageReader implements IPageReader {
if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aBinary))) {
timeBuilder.writeLong(timestamp);
valueBuilder.writeBinary(aBinary);
+ builder.declarePosition();
}
}
break;
@@ -230,7 +236,7 @@ public class PageReader implements IPageReader {
throw new UnSupportedDataTypeException(String.valueOf(dataType));
}
}
- return tsBlockBuilder.build();
+ return builder.build();
}
@Override
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockTest.java
new file mode 100644
index 0000000..fe3a862
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/common/block/TsBlockTest.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.common.block;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.*;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.junit.Assert.*;
+
+public class TsBlockTest {
+
+ private static final double DELTA = 0.000001;
+
+ @Test
+ public void testBooleanTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ boolean[] valueArray = {true, false, false, false, false};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.BOOLEAN));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeBoolean(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof BooleanColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getBoolean(i));
+ }
+ }
+
+ @Test
+ public void testIntTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ int[] valueArray = {10, 20, 30, 40, 50};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeInt(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof IntColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getInt(i));
+ }
+ }
+
+ @Test
+ public void testLongTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ long[] valueArray = {10L, 20L, 30L, 40L, 50L};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT64));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeLong(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof LongColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getLong(i));
+ }
+ }
+
+ @Test
+ public void testFloatTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ float[] valueArray = {10.0f, 20.0f, 30.0f, 40.0f, 50.0f};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.FLOAT));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeFloat(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof FloatColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getFloat(i), DELTA);
+ }
+ }
+
+ @Test
+ public void testDoubleTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ double[] valueArray = {10.0, 20.0, 30.0, 40.0, 50.0};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.DOUBLE));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeDouble(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof DoubleColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getDouble(i), DELTA);
+ }
+ }
+
+ @Test
+ public void testBinaryTsBlock() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ Binary[] valueArray = {
+ new Binary("10"), new Binary("20"), new Binary("30"), new Binary("40"), new Binary("50")
+ };
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ builder.getColumnBuilder(0).writeBinary(valueArray[i]);
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof BinaryColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertFalse(tsBlock.getColumn(0).isNull(i));
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getBinary(i));
+ }
+ }
+
+ @Test
+ public void testIntTsBlockWithNull() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ int[] valueArray = {10, 20, 30, 40, 50};
+ boolean[] isNull = {false, false, true, true, false};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ if (isNull[i]) {
+ builder.getColumnBuilder(0).appendNull();
+
+ } else {
+ builder.getColumnBuilder(0).writeInt(valueArray[i]);
+ }
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof IntColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertEquals(isNull[i], tsBlock.getColumn(0).isNull(i));
+ if (!isNull[i]) {
+ assertEquals(valueArray[i], tsBlock.getColumn(0).getInt(i));
+ }
+ }
+ }
+
+ @Test
+ public void testIntTsBlockWithAllNull() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32));
+ for (long l : timeArray) {
+ builder.getTimeColumnBuilder().writeLong(l);
+ builder.getColumnBuilder(0).appendNull();
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(1, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof RunLengthEncodedColumn);
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertTrue(tsBlock.getColumn(0).isNull(i));
+ }
+ }
+
+ @Test
+ public void testMultiColumnTsBlockWithNull() {
+ long[] timeArray = {1L, 2L, 3L, 4L, 5L};
+ boolean[] booleanValueArray = {true, false, false, false, true};
+ boolean[] booleanIsNull = {true, true, false, true, false};
+ int[] intValueArray = {10, 20, 30, 40, 50};
+ boolean[] intIsNull = {false, true, false, false, true};
+ long[] longValueArray = {100L, 200L, 300L, 400, 500L};
+ boolean[] longIsNull = {true, false, false, true, true};
+ float[] floatValueArray = {1000.0f, 2000.0f, 3000.0f, 4000.0f, 5000.0f};
+ boolean[] floatIsNull = {false, false, true, true, false};
+ double[] doubleValueArray = {10000.0, 20000.0, 30000.0, 40000.0, 50000.0};
+ boolean[] doubleIsNull = {true, false, false, true, false};
+ Binary[] binaryValueArray = {
+ new Binary("19970909"),
+ new Binary("ty"),
+ new Binary("love"),
+ new Binary("zm"),
+ new Binary("19950421")
+ };
+ boolean[] binaryIsNull = {false, false, false, false, false};
+
+ TsBlockBuilder builder =
+ new TsBlockBuilder(
+ Arrays.asList(
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT));
+ for (int i = 0; i < timeArray.length; i++) {
+ builder.getTimeColumnBuilder().writeLong(timeArray[i]);
+ if (booleanIsNull[i]) {
+ builder.getColumnBuilder(0).appendNull();
+ } else {
+ builder.getColumnBuilder(0).writeBoolean(booleanValueArray[i]);
+ }
+ if (intIsNull[i]) {
+ builder.getColumnBuilder(1).appendNull();
+ } else {
+ builder.getColumnBuilder(1).writeInt(intValueArray[i]);
+ }
+ if (longIsNull[i]) {
+ builder.getColumnBuilder(2).appendNull();
+ } else {
+ builder.getColumnBuilder(2).writeLong(longValueArray[i]);
+ }
+ if (floatIsNull[i]) {
+ builder.getColumnBuilder(3).appendNull();
+ } else {
+ builder.getColumnBuilder(3).writeFloat(floatValueArray[i]);
+ }
+ if (doubleIsNull[i]) {
+ builder.getColumnBuilder(4).appendNull();
+ } else {
+ builder.getColumnBuilder(4).writeDouble(doubleValueArray[i]);
+ }
+ if (binaryIsNull[i]) {
+ builder.getColumnBuilder(5).appendNull();
+ } else {
+ builder.getColumnBuilder(5).writeBinary(binaryValueArray[i]);
+ }
+ builder.declarePosition();
+ }
+ TsBlock tsBlock = builder.build();
+ assertEquals(timeArray.length, tsBlock.getPositionCount());
+ assertEquals(6, tsBlock.getValueColumnCount());
+ assertTrue(tsBlock.getColumn(0) instanceof BooleanColumn);
+ assertTrue(tsBlock.getColumn(1) instanceof IntColumn);
+ assertTrue(tsBlock.getColumn(2) instanceof LongColumn);
+ assertTrue(tsBlock.getColumn(3) instanceof FloatColumn);
+ assertTrue(tsBlock.getColumn(4) instanceof DoubleColumn);
+ assertTrue(tsBlock.getColumn(5) instanceof BinaryColumn);
+
+ for (int i = 0; i < timeArray.length; i++) {
+ assertEquals(timeArray[i], tsBlock.getTimeByIndex(i));
+ assertEquals(booleanIsNull[i], tsBlock.getColumn(0).isNull(i));
+ if (!booleanIsNull[i]) {
+ assertEquals(booleanValueArray[i], tsBlock.getColumn(0).getBoolean(i));
+ }
+ assertEquals(intIsNull[i], tsBlock.getColumn(1).isNull(i));
+ if (!intIsNull[i]) {
+ assertEquals(intValueArray[i], tsBlock.getColumn(1).getInt(i));
+ }
+ assertEquals(longIsNull[i], tsBlock.getColumn(2).isNull(i));
+ if (!longIsNull[i]) {
+ assertEquals(longValueArray[i], tsBlock.getColumn(2).getLong(i));
+ }
+ assertEquals(floatIsNull[i], tsBlock.getColumn(3).isNull(i));
+ if (!floatIsNull[i]) {
+ assertEquals(floatValueArray[i], tsBlock.getColumn(3).getFloat(i), DELTA);
+ }
+ assertEquals(doubleIsNull[i], tsBlock.getColumn(4).isNull(i));
+ if (!doubleIsNull[i]) {
+ assertEquals(doubleValueArray[i], tsBlock.getColumn(4).getDouble(i), DELTA);
+ }
+ assertEquals(binaryIsNull[i], tsBlock.getColumn(5).isNull(i));
+ if (!binaryIsNull[i]) {
+ assertEquals(binaryValueArray[i], tsBlock.getColumn(5).getBinary(i));
+ }
+ }
+ }
+}