You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/11/16 06:26:10 UTC

(iotdb) branch master updated: Skip null value row when counting time (#11459)

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 5c438ac89f9 Skip null value row when counting time (#11459)
5c438ac89f9 is described below

commit 5c438ac89f9f3a2734b969934cdabd6ab62f2c0c
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Nov 16 14:26:02 2023 +0800

    Skip null value row when counting time (#11459)
---
 .../aggregation/IoTDBCountTimeAlignedDeviceIT.java | 58 ++++++++++++++++++++
 .../dataregion/memtable/AbstractMemTable.java      |  2 +-
 .../memtable/AlignedWritableMemChunk.java          | 21 ++++++--
 .../db/utils/datastructure/AlignedTVList.java      | 62 +++++++++++++++++++++-
 4 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
index 04b9f5746d4..d1647cd3ef5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountTimeAlignedDeviceIT.java
@@ -74,6 +74,18 @@ public class IoTDBCountTimeAlignedDeviceIT {
         "CREATE ALIGNED TIMESERIES root.aligned.having.d2(s1 INT32, s2 INT32);",
         "INSERT INTO root.aligned.having.d1(time, s1, s2) ALIGNED VALUES(0,0,null), (1,null,1), (2,null,2), (4,4,null), (5,5,5), (7,null,7), (8,8,8), (9,null,9);",
         "INSERT INTO root.aligned.having.d2(time, s1, s2) ALIGNED VALUES(0,null,0), (1,1,null), (2,2,null), (4,null,4), (5,5,5), (7,7,null), (8,8,8), (9,9,null);",
+
+        // test delete
+        "CREATE ALIGNED TIMESERIES root.aligned.delete.d1(s1 INT32, s2 INT32);",
+        "CREATE ALIGNED TIMESERIES root.aligned.delete.d2(s1 INT32, s2 INT32);",
+        "INSERT INTO root.aligned.delete.d1(time, s1, s2) ALIGNED VALUES(1, 1, null), (2, null, 2);",
+        "INSERT INTO root.aligned.delete.d2(time, s1, s2) ALIGNED VALUES(1, null, 1);",
+        "DELETE FROM root.aligned.delete.** where time < 2;",
+        // test null
+        "create aligned timeseries root.aligned.null.d1(s1 double,s2 double,s3 double);",
+        "insert into root.aligned.null.d1(time,s1) values(1,null);",
+        "insert into root.aligned.null.d1(time,s1) values(2,null);",
+        "insert into root.aligned.null.d1(time,s1) values(3,null);",
       };
 
   @BeforeClass
@@ -301,4 +313,50 @@ public class IoTDBCountTimeAlignedDeviceIT {
         expectedHeader,
         retArray);
   }
+
+  @Test
+  public void deleteTest() {
+    // align by time
+    String[] expectedHeader = new String[] {"count_time(*)"};
+    String[] retArray = new String[] {"1,"};
+    resultSetEqualTest(
+        "SELECT COUNT_TIME(*) FROM root.aligned.delete.**;", expectedHeader, retArray);
+
+    expectedHeader = new String[] {"count_time(*)"};
+    retArray = new String[] {"1,"};
+    resultSetEqualTest(
+        "SELECT COUNT_TIME(*) FROM root.aligned.delete.d1, root.aligned.delete.d2;",
+        expectedHeader,
+        retArray);
+
+    // align by device
+    expectedHeader = new String[] {"Device,count_time(*)"};
+    retArray = new String[] {"root.aligned.delete.d1,1,", "root.aligned.delete.d2,0,"};
+    resultSetEqualTest(
+        "select count_time(*) from root.aligned.delete.** align by device;",
+        expectedHeader,
+        retArray);
+
+    expectedHeader = new String[] {"Device,count_time(*)"};
+    retArray = new String[] {"root.aligned.delete.d1,1,", "root.aligned.delete.d2,0,"};
+    resultSetEqualTest(
+        "select count_time(*) from root.aligned.delete.d1,root.aligned.delete.d2 align by device;",
+        expectedHeader,
+        retArray);
+  }
+
+  @Test
+  public void nullRowTest() {
+    // align by time
+    String[] expectedHeader = new String[] {"count_time(*)"};
+    String[] retArray = new String[] {"0,"};
+    resultSetEqualTest("SELECT COUNT_TIME(*) FROM root.aligned.null.**;", expectedHeader, retArray);
+
+    expectedHeader = new String[] {"count_time(*)"};
+    retArray = new String[] {"0,"};
+    resultSetEqualTest(
+        "SELECT COUNT_TIME(*) FROM root.aligned.null.d1, root.aligned.null.d2;",
+        expectedHeader,
+        retArray);
+  }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index e07bbafcc31..16681e7a424 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -229,7 +229,7 @@ public abstract class AbstractMemTable implements IMemTable {
     List<TSDataType> dataTypes = new ArrayList<>();
     for (int i = 0; i < insertRowNode.getMeasurements().length; i++) {
       // Use measurements[i] to ignore failed partial insert
-      if (measurements[i] == null) {
+      if (measurements[i] == null || values[i] == null) {
         schemaList.add(null);
         continue;
       }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 10840917f14..c07cb4fb6a5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -331,6 +331,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
   public void encode(IChunkWriter chunkWriter) {
     AlignedChunkWriterImpl alignedChunkWriter = (AlignedChunkWriterImpl) chunkWriter;
 
+    BitMap rowBitMap = list.getRowBitMap();
     boolean[] timeDuplicateInfo = null;
     List<Integer> pageRange = new ArrayList<>();
     int range = 0;
@@ -345,12 +346,19 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
         range = 0;
       }
 
-      if (sortedRowIndex != list.rowCount() - 1 && time == list.getTime(sortedRowIndex + 1)) {
+      int nextRowIndex = sortedRowIndex + 1;
+      while (nextRowIndex < list.rowCount()
+          && rowBitMap != null
+          && rowBitMap.isMarked(nextRowIndex)) {
+        nextRowIndex++;
+      }
+      if (nextRowIndex != list.rowCount() && time == list.getTime(nextRowIndex)) {
         if (Objects.isNull(timeDuplicateInfo)) {
           timeDuplicateInfo = new boolean[list.rowCount()];
         }
         timeDuplicateInfo[sortedRowIndex] = true;
       }
+      sortedRowIndex = nextRowIndex - 1;
     }
 
     if (range != 0) {
@@ -367,11 +375,14 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
             && lastValidPointIndexForTimeDupCheck[columnIndex] == null) {
           lastValidPointIndexForTimeDupCheck[columnIndex] = new Pair<>(Long.MIN_VALUE, null);
         }
+        TSDataType tsDataType = dataTypes.get(columnIndex);
         for (int sortedRowIndex = pageRange.get(pageNum * 2);
             sortedRowIndex <= pageRange.get(pageNum * 2 + 1);
             sortedRowIndex++) {
-          TSDataType tsDataType = dataTypes.get(columnIndex);
-
+          // skip empty row
+          if (rowBitMap != null && rowBitMap.isMarked(list.getValueIndex(sortedRowIndex))) {
+            continue;
+          }
           // skip time duplicated rows
           long time = list.getTime(sortedRowIndex);
           if (Objects.nonNull(timeDuplicateInfo)) {
@@ -450,6 +461,10 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
       for (int sortedRowIndex = pageRange.get(pageNum * 2);
           sortedRowIndex <= pageRange.get(pageNum * 2 + 1);
           sortedRowIndex++) {
+        // skip empty row
+        if (rowBitMap != null && rowBitMap.isMarked(list.getValueIndex(sortedRowIndex))) {
+          continue;
+        }
         if (Objects.isNull(timeDuplicateInfo) || !timeDuplicateInfo[sortedRowIndex]) {
           times[pointsInPage++] = list.getTime(sortedRowIndex);
         }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 32b2111a74c..0d456dd3e92 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -74,6 +74,9 @@ public abstract class AlignedTVList extends TVList {
   // If a sensor chunk size of Text datatype reaches the threshold, this flag will be set true
   boolean reachMaxChunkSizeFlag;
 
+  // not null when constructed by queries
+  BitMap rowBitMap;
+
   AlignedTVList(List<TSDataType> types) {
     super();
     indices = new ArrayList<>(types.size());
@@ -125,6 +128,7 @@ public abstract class AlignedTVList extends TVList {
     alignedTvList.values = values;
     alignedTvList.bitMaps = bitMaps;
     alignedTvList.rowCount = this.rowCount;
+    alignedTvList.rowBitMap = getRowBitMap();
     return alignedTvList;
   }
 
@@ -910,8 +914,15 @@ public abstract class AlignedTVList extends TVList {
     boolean[] timeDuplicateInfo = null;
     // time column
     for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) {
-      if (sortedRowIndex == rowCount - 1
-          || getTime(sortedRowIndex) != getTime(sortedRowIndex + 1)) {
+      // skip empty row
+      if (rowBitMap != null && rowBitMap.isMarked(sortedRowIndex)) {
+        continue;
+      }
+      int nextRowIndex = sortedRowIndex + 1;
+      while (nextRowIndex < rowCount && rowBitMap != null && rowBitMap.isMarked(nextRowIndex)) {
+        nextRowIndex++;
+      }
+      if (nextRowIndex == rowCount || getTime(sortedRowIndex) != getTime(nextRowIndex)) {
         timeBuilder.writeLong(getTime(sortedRowIndex));
         validRowCount++;
       } else {
@@ -920,6 +931,7 @@ public abstract class AlignedTVList extends TVList {
         }
         timeDuplicateInfo[sortedRowIndex] = true;
       }
+      sortedRowIndex = nextRowIndex - 1;
     }
 
     // value columns
@@ -932,6 +944,10 @@ public abstract class AlignedTVList extends TVList {
       }
       ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
       for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) {
+        // skip empty row
+        if (rowBitMap != null && rowBitMap.isMarked(getValueIndex(sortedRowIndex))) {
+          continue;
+        }
         // skip time duplicated rows
         if (Objects.nonNull(timeDuplicateInfo)) {
           if (!isNullValue(getValueIndex(sortedRowIndex), columnIndex)) {
@@ -1199,4 +1215,46 @@ public abstract class AlignedTVList extends TVList {
     tvList.putAlignedValues(times, values, bitMaps, 0, rowCount);
     return tvList;
   }
+
+  public BitMap getRowBitMap() {
+    // row exists when any column value exists
+    if (bitMaps == null) {
+      return null;
+    }
+    for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
+      if (values.get(columnIndex) != null && bitMaps.get(columnIndex) == null) {
+        return null;
+      }
+    }
+
+    byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1];
+    for (int row = 0; row < rowCount; row += Byte.SIZE) {
+      boolean isFirstColumn = true;
+      byte rowBits = 0x00;
+      for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
+        List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
+        byte columnBits;
+        if (values.get(columnIndex) == null) {
+          columnBits = (byte) 0xFF;
+        } else if (columnBitMaps == null || columnBitMaps.get(row / ARRAY_SIZE) == null) {
+          // row exists when any column value exists
+          rowBits = 0x00;
+          break;
+        } else {
+          columnBits =
+              columnBitMaps.get(row / ARRAY_SIZE).getByteArray()[(row % ARRAY_SIZE) / Byte.SIZE];
+        }
+        // set row to null when all column values are null
+        if (isFirstColumn) {
+          rowBits = columnBits;
+          isFirstColumn = false;
+        } else {
+          rowBits &= columnBits;
+        }
+      }
+      rowBitsArr[row / Byte.SIZE] = rowBits;
+    }
+
+    return new BitMap(rowCount, rowBitsArr);
+  }
 }