You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/10/22 03:27:54 UTC

(iotdb) 01/01: Fix ignored bitmap of attribute in writeTablet

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch fix_ignored_bitmap_in_attr_write
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 374a0e7c0b12a72a624d491770c987a17c6799c7
Author: Tian Jiang <jt...@163.com>
AuthorDate: Tue Oct 22 11:28:51 2024 +0800

    Fix ignored bitmap of attribute in writeTablet
---
 .../PipeConvertedInsertTabletStatement.java        |   2 +-
 .../plan/relational/sql/ast/InsertTablet.java      |   4 +-
 .../plan/statement/crud/InsertTabletStatement.java |  44 ++++----
 .../plan/relational/sql/ast/InsertTabletTest.java  | 111 +++++++++++++++++++++
 4 files changed, 141 insertions(+), 20 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
index a5013725061..50dfbad00de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
@@ -44,7 +44,7 @@ public class PipeConvertedInsertTabletStatement extends InsertTabletStatement {
     dataTypes = insertTabletStatement.getDataTypes();
     // InsertTabletStatement
     times = insertTabletStatement.getTimes();
-    bitMaps = insertTabletStatement.getBitMaps();
+    nullBitMaps = insertTabletStatement.getBitMaps();
     columns = insertTabletStatement.getColumns();
     rowCount = insertTabletStatement.getRowCount();
   }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
index 4feecf2807c..7f56e3bac06 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
@@ -82,7 +82,9 @@ public class InsertTablet extends WrappedInsertStatement {
       Object[] attrValues = new Object[attrColumnIndices.size()];
       for (int j = 0; j < attrColumnIndices.size(); j++) {
         final int columnIndex = attrColumnIndices.get(j);
-        attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
+        if (!insertTabletStatement.isNull(i, j)) {
+          attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
+        }
       }
       result.add(attrValues);
     }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index 6a70a5e5493..fe632bc4978 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -66,7 +66,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
   private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported.";
 
   protected long[] times; // times should be sorted. It is done in the session API.
-  protected BitMap[] bitMaps;
+  protected BitMap[] nullBitMaps;
   protected Object[] columns;
 
   private IDeviceID[] deviceIDs;
@@ -122,11 +122,11 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
   }
 
   public BitMap[] getBitMaps() {
-    return bitMaps;
+    return nullBitMaps;
   }
 
   public void setBitMaps(BitMap[] bitMaps) {
-    this.bitMaps = bitMaps;
+    this.nullBitMaps = bitMaps;
   }
 
   public long[] getTimes() {
@@ -278,8 +278,8 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
         measurements[i] = pairList.get(i).left;
         measurementSchemas[i] = this.measurementSchemas[realIndex];
         dataTypes[i] = this.dataTypes[realIndex];
-        if (this.bitMaps != null) {
-          copiedBitMaps[i] = this.bitMaps[realIndex];
+        if (this.nullBitMaps != null) {
+          copiedBitMaps[i] = this.nullBitMaps[realIndex];
         }
         if (this.measurementIsAligned != null) {
           statement.setAligned(this.measurementIsAligned[realIndex]);
@@ -289,7 +289,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
       statement.setMeasurements(measurements);
       statement.setMeasurementSchemas(measurementSchemas);
       statement.setDataTypes(dataTypes);
-      if (this.bitMaps != null) {
+      if (this.nullBitMaps != null) {
         statement.setBitMaps(copiedBitMaps);
       }
       statement.setFailedMeasurementIndex2Info(failedMeasurementIndex2Info);
@@ -461,8 +461,9 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
       deviceIdSegments[0] = this.getTableName();
       for (int i = 0; i < getIdColumnIndices().size(); i++) {
         final Integer columnIndex = getIdColumnIndices().get(i);
-        Object idSeg = ((Object[]) columns[columnIndex])[rowIdx];
-        deviceIdSegments[i + 1] = idSeg != null ? idSeg.toString() : null;
+        boolean isNull = isNull(rowIdx, i);
+        deviceIdSegments[i + 1] =
+            isNull ? null : ((Object[]) columns[columnIndex])[rowIdx].toString();
       }
       deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
     }
@@ -474,21 +475,21 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
   public void insertColumn(int pos, ColumnSchema columnSchema) {
     super.insertColumn(pos, columnSchema);
 
-    if (bitMaps == null) {
-      bitMaps = new BitMap[measurements.length];
-      bitMaps[pos] = new BitMap(rowCount);
+    if (nullBitMaps == null) {
+      nullBitMaps = new BitMap[measurements.length];
+      nullBitMaps[pos] = new BitMap(rowCount);
       for (int i = 0; i < rowCount; i++) {
-        bitMaps[pos].mark(i);
+        nullBitMaps[pos].mark(i);
       }
     } else {
-      BitMap[] tmpBitmaps = new BitMap[bitMaps.length + 1];
-      System.arraycopy(bitMaps, 0, tmpBitmaps, 0, pos);
+      BitMap[] tmpBitmaps = new BitMap[nullBitMaps.length + 1];
+      System.arraycopy(nullBitMaps, 0, tmpBitmaps, 0, pos);
       tmpBitmaps[pos] = new BitMap(rowCount);
       for (int i = 0; i < rowCount; i++) {
         tmpBitmaps[pos].mark(i);
       }
-      System.arraycopy(bitMaps, pos, tmpBitmaps, pos + 1, bitMaps.length - pos);
-      bitMaps = tmpBitmaps;
+      System.arraycopy(nullBitMaps, pos, tmpBitmaps, pos + 1, nullBitMaps.length - pos);
+      nullBitMaps = tmpBitmaps;
     }
 
     Object[] tmpColumns = new Object[columns.length + 1];
@@ -507,10 +508,17 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
   @Override
   public void swapColumn(int src, int target) {
     super.swapColumn(src, target);
-    if (bitMaps != null) {
-      CommonUtils.swapArray(bitMaps, src, target);
+    if (nullBitMaps != null) {
+      CommonUtils.swapArray(nullBitMaps, src, target);
     }
     CommonUtils.swapArray(columns, src, target);
     deviceIDs = null;
   }
+
+  public boolean isNull(int row, int col) {
+    if (nullBitMaps == null || nullBitMaps[col] == null) {
+      return false;
+    }
+    return nullBitMaps[col].isMarked(row);
+  }
 }
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
new file mode 100644
index 00000000000..9b2127bccfc
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
@@ -0,0 +1,111 @@
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class InsertTabletTest {
+
+  @Test
+  public void testWithNull() {
+    InsertTabletStatement innerStmt = new InsertTabletStatement();
+    innerStmt.setDevicePath(new PartialPath("table1", false));
+    innerStmt.setTimes(new long[] {1, 2, 3, 4});
+    innerStmt.setRowCount(4);
+    innerStmt.setMeasurements(
+        new String[] {"deviceId_1", "deviceId_2", "attr1", "attr2", "measurement"});
+    innerStmt.setColumnCategories(
+        new TsTableColumnCategory[] {
+          TsTableColumnCategory.ID, TsTableColumnCategory.ID,
+          TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.ATTRIBUTE,
+          TsTableColumnCategory.MEASUREMENT
+        });
+    innerStmt.setDataTypes(
+        new TSDataType[] {
+          TSDataType.STRING,
+          TSDataType.STRING,
+          TSDataType.STRING,
+          TSDataType.STRING,
+          TSDataType.STRING
+        });
+    innerStmt.setColumns(
+        new Object[] {
+          new Binary[] {
+            new Binary("id1_1", StandardCharsets.UTF_8),
+            new Binary("id2_1", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            Binary.EMPTY_VALUE,
+          },
+          new Binary[] {
+            new Binary("id1_2", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            new Binary("id3_2", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+          },
+          new Binary[] {
+            new Binary("attr1_1", StandardCharsets.UTF_8),
+            new Binary("attr2_1", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            Binary.EMPTY_VALUE,
+          },
+          new Binary[] {
+            new Binary("attr1_2", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            new Binary("attr3_2", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+          },
+          new Binary[] {
+            new Binary("m1", StandardCharsets.UTF_8),
+            new Binary("m2", StandardCharsets.UTF_8),
+            new Binary("m3", StandardCharsets.UTF_8),
+            new Binary("m4", StandardCharsets.UTF_8)
+          },
+        });
+    innerStmt.setBitMaps(
+        new BitMap[] {
+          new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+          new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+          new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+          new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+          new BitMap(4, new byte[] {0x00}),
+        });
+
+    InsertTablet insertTablet = new InsertTablet(innerStmt, null);
+    assertEquals(Arrays.asList("attr1", "attr2"), insertTablet.getAttributeColumnNameList());
+    List<Object[]> attributeValueList = insertTablet.getAttributeValueList();
+    assertEquals(4, attributeValueList.size());
+    assertArrayEquals(
+        new Object[] {
+          new Binary("attr1_1", StandardCharsets.UTF_8),
+          new Binary("attr1_2", StandardCharsets.UTF_8)
+        },
+        attributeValueList.get(0));
+    assertArrayEquals(
+        new Object[] {new Binary("attr2_1", StandardCharsets.UTF_8), null},
+        attributeValueList.get(1));
+    assertArrayEquals(
+        new Object[] {null, new Binary("attr3_2", StandardCharsets.UTF_8)},
+        attributeValueList.get(2));
+    assertArrayEquals(new Object[] {null, null}, attributeValueList.get(3));
+
+    List<Object[]> deviceIdList = insertTablet.getDeviceIdList();
+    assertEquals(4, deviceIdList.size());
+    assertArrayEquals(new Object[] {"id1_1", "id1_2"}, deviceIdList.get(0));
+    // notice: trailing nulls are removed
+    assertArrayEquals(new Object[] {"id2_1"}, deviceIdList.get(1));
+    assertArrayEquals(new Object[] {null, "id3_2"}, deviceIdList.get(2));
+    assertArrayEquals(new Object[] {}, deviceIdList.get(3));
+  }
+}