You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/11/16 06:26:10 UTC
(iotdb) branch master updated: Skip null value row when counting time (#11459)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5c438ac89f9 Skip null value row when counting time (#11459)
5c438ac89f9 is described below
commit 5c438ac89f9f3a2734b969934cdabd6ab62f2c0c
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Nov 16 14:26:02 2023 +0800
Skip null value row when counting time (#11459)
---
.../aggregation/IoTDBCountTimeAlignedDeviceIT.java | 58 ++++++++++++++++++++
.../dataregion/memtable/AbstractMemTable.java | 2 +-
.../memtable/AlignedWritableMemChunk.java | 21 ++++++--
.../db/utils/datastructure/AlignedTVList.java | 62 +++++++++++++++++++++-
4 files changed, 137 insertions(+), 6 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
index 04b9f5746d4..d1647cd3ef5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
@@ -74,6 +74,18 @@ public class IoTDBCountTimeAlignedDeviceIT {
"CREATE ALIGNED TIMESERIES root.aligned.having.d2(s1 INT32, s2 INT32);",
"INSERT INTO root.aligned.having.d1(time, s1, s2) ALIGNED VALUES(0,0,null), (1,null,1), (2,null,2), (4,4,null), (5,5,5), (7,null,7), (8,8,8), (9,null,9);",
"INSERT INTO root.aligned.having.d2(time, s1, s2) ALIGNED VALUES(0,null,0), (1,1,null), (2,2,null), (4,null,4), (5,5,5), (7,7,null), (8,8,8), (9,9,null);",
+
+ // test delete
+ "CREATE ALIGNED TIMESERIES root.aligned.delete.d1(s1 INT32, s2 INT32);",
+ "CREATE ALIGNED TIMESERIES root.aligned.delete.d2(s1 INT32, s2 INT32);",
+ "INSERT INTO root.aligned.delete.d1(time, s1, s2) ALIGNED VALUES(1, 1, null), (2, null, 2);",
+ "INSERT INTO root.aligned.delete.d2(time, s1, s2) ALIGNED VALUES(1, null, 1);",
+ "DELETE FROM root.aligned.delete.** where time < 2;",
+ // test null
+ "create aligned timeseries root.aligned.null.d1(s1 double,s2 double,s3 double);",
+ "insert into root.aligned.null.d1(time,s1) values(1,null);",
+ "insert into root.aligned.null.d1(time,s1) values(2,null);",
+ "insert into root.aligned.null.d1(time,s1) values(3,null);",
};
@BeforeClass
@@ -301,4 +313,50 @@ public class IoTDBCountTimeAlignedDeviceIT {
expectedHeader,
retArray);
}
+
+ @Test
+ public void deleteTest() {
+ // align by time
+ String[] expectedHeader = new String[] {"count_time(*)"};
+ String[] retArray = new String[] {"1,"};
+ resultSetEqualTest(
+ "SELECT COUNT_TIME(*) FROM root.aligned.delete.**;", expectedHeader, retArray);
+
+ expectedHeader = new String[] {"count_time(*)"};
+ retArray = new String[] {"1,"};
+ resultSetEqualTest(
+ "SELECT COUNT_TIME(*) FROM root.aligned.delete.d1, root.aligned.delete.d2;",
+ expectedHeader,
+ retArray);
+
+ // align by device
+ expectedHeader = new String[] {"Device,count_time(*)"};
+ retArray = new String[] {"root.aligned.delete.d1,1,", "root.aligned.delete.d2,0,"};
+ resultSetEqualTest(
+ "select count_time(*) from root.aligned.delete.** align by device;",
+ expectedHeader,
+ retArray);
+
+ expectedHeader = new String[] {"Device,count_time(*)"};
+ retArray = new String[] {"root.aligned.delete.d1,1,", "root.aligned.delete.d2,0,"};
+ resultSetEqualTest(
+ "select count_time(*) from root.aligned.delete.d1,root.aligned.delete.d2 align by device;",
+ expectedHeader,
+ retArray);
+ }
+
+ @Test
+ public void nullRowTest() {
+ // align by time
+ String[] expectedHeader = new String[] {"count_time(*)"};
+ String[] retArray = new String[] {"0,"};
+ resultSetEqualTest("SELECT COUNT_TIME(*) FROM root.aligned.null.**;", expectedHeader, retArray);
+
+ expectedHeader = new String[] {"count_time(*)"};
+ retArray = new String[] {"0,"};
+ resultSetEqualTest(
+ "SELECT COUNT_TIME(*) FROM root.aligned.null.d1, root.aligned.null.d2;",
+ expectedHeader,
+ retArray);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index e07bbafcc31..16681e7a424 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -229,7 +229,7 @@ public abstract class AbstractMemTable implements IMemTable {
List<TSDataType> dataTypes = new ArrayList<>();
for (int i = 0; i < insertRowNode.getMeasurements().length; i++) {
// Use measurements[i] to ignore failed partial insert
- if (measurements[i] == null) {
+ if (measurements[i] == null || values[i] == null) {
schemaList.add(null);
continue;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 10840917f14..c07cb4fb6a5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -331,6 +331,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
public void encode(IChunkWriter chunkWriter) {
AlignedChunkWriterImpl alignedChunkWriter = (AlignedChunkWriterImpl) chunkWriter;
+ BitMap rowBitMap = list.getRowBitMap();
boolean[] timeDuplicateInfo = null;
List<Integer> pageRange = new ArrayList<>();
int range = 0;
@@ -345,12 +346,19 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
range = 0;
}
- if (sortedRowIndex != list.rowCount() - 1 && time == list.getTime(sortedRowIndex + 1)) {
+ int nextRowIndex = sortedRowIndex + 1;
+ while (nextRowIndex < list.rowCount()
+ && rowBitMap != null
+ && rowBitMap.isMarked(nextRowIndex)) {
+ nextRowIndex++;
+ }
+ if (nextRowIndex != list.rowCount() && time == list.getTime(nextRowIndex)) {
if (Objects.isNull(timeDuplicateInfo)) {
timeDuplicateInfo = new boolean[list.rowCount()];
}
timeDuplicateInfo[sortedRowIndex] = true;
}
+ sortedRowIndex = nextRowIndex - 1;
}
if (range != 0) {
@@ -367,11 +375,14 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
&& lastValidPointIndexForTimeDupCheck[columnIndex] == null) {
lastValidPointIndexForTimeDupCheck[columnIndex] = new Pair<>(Long.MIN_VALUE, null);
}
+ TSDataType tsDataType = dataTypes.get(columnIndex);
for (int sortedRowIndex = pageRange.get(pageNum * 2);
sortedRowIndex <= pageRange.get(pageNum * 2 + 1);
sortedRowIndex++) {
- TSDataType tsDataType = dataTypes.get(columnIndex);
-
+ // skip empty row
+ if (rowBitMap != null && rowBitMap.isMarked(list.getValueIndex(sortedRowIndex))) {
+ continue;
+ }
// skip time duplicated rows
long time = list.getTime(sortedRowIndex);
if (Objects.nonNull(timeDuplicateInfo)) {
@@ -450,6 +461,10 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
for (int sortedRowIndex = pageRange.get(pageNum * 2);
sortedRowIndex <= pageRange.get(pageNum * 2 + 1);
sortedRowIndex++) {
+ // skip empty row
+ if (rowBitMap != null && rowBitMap.isMarked(list.getValueIndex(sortedRowIndex))) {
+ continue;
+ }
if (Objects.isNull(timeDuplicateInfo) || !timeDuplicateInfo[sortedRowIndex]) {
times[pointsInPage++] = list.getTime(sortedRowIndex);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 32b2111a74c..0d456dd3e92 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -74,6 +74,9 @@ public abstract class AlignedTVList extends TVList {
// If a sensor chunk size of Text datatype reaches the threshold, this flag will be set true
boolean reachMaxChunkSizeFlag;
+ // not null when constructed by queries
+ BitMap rowBitMap;
+
AlignedTVList(List<TSDataType> types) {
super();
indices = new ArrayList<>(types.size());
@@ -125,6 +128,7 @@ public abstract class AlignedTVList extends TVList {
alignedTvList.values = values;
alignedTvList.bitMaps = bitMaps;
alignedTvList.rowCount = this.rowCount;
+ alignedTvList.rowBitMap = getRowBitMap();
return alignedTvList;
}
@@ -910,8 +914,15 @@ public abstract class AlignedTVList extends TVList {
boolean[] timeDuplicateInfo = null;
// time column
for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) {
- if (sortedRowIndex == rowCount - 1
- || getTime(sortedRowIndex) != getTime(sortedRowIndex + 1)) {
+ // skip empty row
+ if (rowBitMap != null && rowBitMap.isMarked(sortedRowIndex)) {
+ continue;
+ }
+ int nextRowIndex = sortedRowIndex + 1;
+ while (nextRowIndex < rowCount && rowBitMap != null && rowBitMap.isMarked(nextRowIndex)) {
+ nextRowIndex++;
+ }
+ if (nextRowIndex == rowCount || getTime(sortedRowIndex) != getTime(nextRowIndex)) {
timeBuilder.writeLong(getTime(sortedRowIndex));
validRowCount++;
} else {
@@ -920,6 +931,7 @@ public abstract class AlignedTVList extends TVList {
}
timeDuplicateInfo[sortedRowIndex] = true;
}
+ sortedRowIndex = nextRowIndex - 1;
}
// value columns
@@ -932,6 +944,10 @@ public abstract class AlignedTVList extends TVList {
}
ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) {
+ // skip empty row
+ if (rowBitMap != null && rowBitMap.isMarked(getValueIndex(sortedRowIndex))) {
+ continue;
+ }
// skip time duplicated rows
if (Objects.nonNull(timeDuplicateInfo)) {
if (!isNullValue(getValueIndex(sortedRowIndex), columnIndex)) {
@@ -1199,4 +1215,46 @@ public abstract class AlignedTVList extends TVList {
tvList.putAlignedValues(times, values, bitMaps, 0, rowCount);
return tvList;
}
+
+ public BitMap getRowBitMap() {
+ // row exists when any column value exists
+ if (bitMaps == null) {
+ return null;
+ }
+ for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
+ if (values.get(columnIndex) != null && bitMaps.get(columnIndex) == null) {
+ return null;
+ }
+ }
+
+ byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1];
+ for (int row = 0; row < rowCount; row += Byte.SIZE) {
+ boolean isFirstColumn = true;
+ byte rowBits = 0x00;
+ for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
+ List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
+ byte columnBits;
+ if (values.get(columnIndex) == null) {
+ columnBits = (byte) 0xFF;
+ } else if (columnBitMaps == null || columnBitMaps.get(row / ARRAY_SIZE) == null) {
+ // row exists when any column value exists
+ rowBits = 0x00;
+ break;
+ } else {
+ columnBits =
+ columnBitMaps.get(row / ARRAY_SIZE).getByteArray()[(row % ARRAY_SIZE) / Byte.SIZE];
+ }
+ // set row to null when all column values are null
+ if (isFirstColumn) {
+ rowBits = columnBits;
+ isFirstColumn = false;
+ } else {
+ rowBits &= columnBits;
+ }
+ }
+ rowBitsArr[row / Byte.SIZE] = rowBits;
+ }
+
+ return new BitMap(rowCount, rowBitsArr);
+ }
}