You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/10/22 03:27:54 UTC
(iotdb) 01/01: Fix ignored bitmap of attribute in writeTablet
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch fix_ignored_bitmap_in_attr_write
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 374a0e7c0b12a72a624d491770c987a17c6799c7
Author: Tian Jiang <jt...@163.com>
AuthorDate: Tue Oct 22 11:28:51 2024 +0800
Fix ignored bitmap of attribute in writeTablet
---
.../PipeConvertedInsertTabletStatement.java | 2 +-
.../plan/relational/sql/ast/InsertTablet.java | 4 +-
.../plan/statement/crud/InsertTabletStatement.java | 44 ++++----
.../plan/relational/sql/ast/InsertTabletTest.java | 111 +++++++++++++++++++++
4 files changed, 141 insertions(+), 20 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
index a5013725061..50dfbad00de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
@@ -44,7 +44,7 @@ public class PipeConvertedInsertTabletStatement extends InsertTabletStatement {
dataTypes = insertTabletStatement.getDataTypes();
// InsertTabletStatement
times = insertTabletStatement.getTimes();
- bitMaps = insertTabletStatement.getBitMaps();
+ nullBitMaps = insertTabletStatement.getBitMaps();
columns = insertTabletStatement.getColumns();
rowCount = insertTabletStatement.getRowCount();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
index 4feecf2807c..7f56e3bac06 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
@@ -82,7 +82,9 @@ public class InsertTablet extends WrappedInsertStatement {
Object[] attrValues = new Object[attrColumnIndices.size()];
for (int j = 0; j < attrColumnIndices.size(); j++) {
final int columnIndex = attrColumnIndices.get(j);
- attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
+ if (!insertTabletStatement.isNull(i, j)) {
+ attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
+ }
}
result.add(attrValues);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index 6a70a5e5493..fe632bc4978 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -66,7 +66,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported.";
protected long[] times; // times should be sorted. It is done in the session API.
- protected BitMap[] bitMaps;
+ protected BitMap[] nullBitMaps;
protected Object[] columns;
private IDeviceID[] deviceIDs;
@@ -122,11 +122,11 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
}
public BitMap[] getBitMaps() {
- return bitMaps;
+ return nullBitMaps;
}
public void setBitMaps(BitMap[] bitMaps) {
- this.bitMaps = bitMaps;
+ this.nullBitMaps = bitMaps;
}
public long[] getTimes() {
@@ -278,8 +278,8 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
measurements[i] = pairList.get(i).left;
measurementSchemas[i] = this.measurementSchemas[realIndex];
dataTypes[i] = this.dataTypes[realIndex];
- if (this.bitMaps != null) {
- copiedBitMaps[i] = this.bitMaps[realIndex];
+ if (this.nullBitMaps != null) {
+ copiedBitMaps[i] = this.nullBitMaps[realIndex];
}
if (this.measurementIsAligned != null) {
statement.setAligned(this.measurementIsAligned[realIndex]);
@@ -289,7 +289,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
statement.setMeasurements(measurements);
statement.setMeasurementSchemas(measurementSchemas);
statement.setDataTypes(dataTypes);
- if (this.bitMaps != null) {
+ if (this.nullBitMaps != null) {
statement.setBitMaps(copiedBitMaps);
}
statement.setFailedMeasurementIndex2Info(failedMeasurementIndex2Info);
@@ -461,8 +461,9 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
deviceIdSegments[0] = this.getTableName();
for (int i = 0; i < getIdColumnIndices().size(); i++) {
final Integer columnIndex = getIdColumnIndices().get(i);
- Object idSeg = ((Object[]) columns[columnIndex])[rowIdx];
- deviceIdSegments[i + 1] = idSeg != null ? idSeg.toString() : null;
+ boolean isNull = isNull(rowIdx, i);
+ deviceIdSegments[i + 1] =
+ isNull ? null : ((Object[]) columns[columnIndex])[rowIdx].toString();
}
deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
}
@@ -474,21 +475,21 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
public void insertColumn(int pos, ColumnSchema columnSchema) {
super.insertColumn(pos, columnSchema);
- if (bitMaps == null) {
- bitMaps = new BitMap[measurements.length];
- bitMaps[pos] = new BitMap(rowCount);
+ if (nullBitMaps == null) {
+ nullBitMaps = new BitMap[measurements.length];
+ nullBitMaps[pos] = new BitMap(rowCount);
for (int i = 0; i < rowCount; i++) {
- bitMaps[pos].mark(i);
+ nullBitMaps[pos].mark(i);
}
} else {
- BitMap[] tmpBitmaps = new BitMap[bitMaps.length + 1];
- System.arraycopy(bitMaps, 0, tmpBitmaps, 0, pos);
+ BitMap[] tmpBitmaps = new BitMap[nullBitMaps.length + 1];
+ System.arraycopy(nullBitMaps, 0, tmpBitmaps, 0, pos);
tmpBitmaps[pos] = new BitMap(rowCount);
for (int i = 0; i < rowCount; i++) {
tmpBitmaps[pos].mark(i);
}
- System.arraycopy(bitMaps, pos, tmpBitmaps, pos + 1, bitMaps.length - pos);
- bitMaps = tmpBitmaps;
+ System.arraycopy(nullBitMaps, pos, tmpBitmaps, pos + 1, nullBitMaps.length - pos);
+ nullBitMaps = tmpBitmaps;
}
Object[] tmpColumns = new Object[columns.length + 1];
@@ -507,10 +508,17 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
@Override
public void swapColumn(int src, int target) {
super.swapColumn(src, target);
- if (bitMaps != null) {
- CommonUtils.swapArray(bitMaps, src, target);
+ if (nullBitMaps != null) {
+ CommonUtils.swapArray(nullBitMaps, src, target);
}
CommonUtils.swapArray(columns, src, target);
deviceIDs = null;
}
+
+ public boolean isNull(int row, int col) {
+ if (nullBitMaps == null || nullBitMaps[col] == null) {
+ return false;
+ }
+ return nullBitMaps[col].isMarked(row);
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
new file mode 100644
index 00000000000..9b2127bccfc
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
@@ -0,0 +1,111 @@
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class InsertTabletTest {
+
+ @Test
+ public void testWithNull() {
+ InsertTabletStatement innerStmt = new InsertTabletStatement();
+ innerStmt.setDevicePath(new PartialPath("table1", false));
+ innerStmt.setTimes(new long[] {1, 2, 3, 4});
+ innerStmt.setRowCount(4);
+ innerStmt.setMeasurements(
+ new String[] {"deviceId_1", "deviceId_2", "attr1", "attr2", "measurement"});
+ innerStmt.setColumnCategories(
+ new TsTableColumnCategory[] {
+ TsTableColumnCategory.ID, TsTableColumnCategory.ID,
+ TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.ATTRIBUTE,
+ TsTableColumnCategory.MEASUREMENT
+ });
+ innerStmt.setDataTypes(
+ new TSDataType[] {
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING
+ });
+ innerStmt.setColumns(
+ new Object[] {
+ new Binary[] {
+ new Binary("id1_1", StandardCharsets.UTF_8),
+ new Binary("id2_1", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ Binary.EMPTY_VALUE,
+ },
+ new Binary[] {
+ new Binary("id1_2", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ new Binary("id3_2", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ },
+ new Binary[] {
+ new Binary("attr1_1", StandardCharsets.UTF_8),
+ new Binary("attr2_1", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ Binary.EMPTY_VALUE,
+ },
+ new Binary[] {
+ new Binary("attr1_2", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ new Binary("attr3_2", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ },
+ new Binary[] {
+ new Binary("m1", StandardCharsets.UTF_8),
+ new Binary("m2", StandardCharsets.UTF_8),
+ new Binary("m3", StandardCharsets.UTF_8),
+ new Binary("m4", StandardCharsets.UTF_8)
+ },
+ });
+ innerStmt.setBitMaps(
+ new BitMap[] {
+ new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+ new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+ new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+ new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+ new BitMap(4, new byte[] {0x00}),
+ });
+
+ InsertTablet insertTablet = new InsertTablet(innerStmt, null);
+ assertEquals(Arrays.asList("attr1", "attr2"), insertTablet.getAttributeColumnNameList());
+ List<Object[]> attributeValueList = insertTablet.getAttributeValueList();
+ assertEquals(4, attributeValueList.size());
+ assertArrayEquals(
+ new Object[] {
+ new Binary("attr1_1", StandardCharsets.UTF_8),
+ new Binary("attr1_2", StandardCharsets.UTF_8)
+ },
+ attributeValueList.get(0));
+ assertArrayEquals(
+ new Object[] {new Binary("attr2_1", StandardCharsets.UTF_8), null},
+ attributeValueList.get(1));
+ assertArrayEquals(
+ new Object[] {null, new Binary("attr3_2", StandardCharsets.UTF_8)},
+ attributeValueList.get(2));
+ assertArrayEquals(new Object[] {null, null}, attributeValueList.get(3));
+
+ List<Object[]> deviceIdList = insertTablet.getDeviceIdList();
+ assertEquals(4, deviceIdList.size());
+ assertArrayEquals(new Object[] {"id1_1", "id1_2"}, deviceIdList.get(0));
+ // notice: trailing nulls are removed
+ assertArrayEquals(new Object[] {"id2_1"}, deviceIdList.get(1));
+ assertArrayEquals(new Object[] {null, "id3_2"}, deviceIdList.get(2));
+ assertArrayEquals(new Object[] {}, deviceIdList.get(3));
+ }
+}