You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/07/22 09:36:47 UTC
(iotdb) branch TableModelIngestion2 created (now 15b428e648c)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch TableModelIngestion2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 15b428e648c fit IT & incorrect tableSchema during validation & get TsTable from & fix npe in partial insertion
This branch includes the following new commits:
new 15b428e648c fit IT & incorrect tableSchema during validation & get TsTable from & fix npe in partial insertion
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: fit IT & incorrect tableSchema during validation & get TsTable from & fix npe in partial insertion
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch TableModelIngestion2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 15b428e648c052202c896b53e01e095fe5b33ce4
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Jul 22 17:36:38 2024 +0800
fit IT & incorrect tableSchema during validation & get TsTable from & fix npe in partial insertion
---
.../iotdb/session/it/IoTDBSessionRelationalIT.java | 126 +++++++++++++++++----
.../fetcher/TableHeaderSchemaValidator.java | 2 +-
.../relational/sql/ast/WrappedInsertStatement.java | 27 +++--
3 files changed, 119 insertions(+), 36 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
index 3a06c9fe433..1fcae69a1e7 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.Tablet.ColumnType;
@@ -43,9 +42,7 @@ import org.junit.runner.RunWith;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
@@ -75,7 +72,77 @@ public class IoTDBSessionRelationalIT {
// for manual debugging
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException {
- insertRelationalTabletPerformanceTest();
+ try (ISession session =
+ new Session.Builder().host("127.0.0.1").port(6667).sqlDialect(TABLE_SQL_DIALECT).build()) {
+ session.open();
+
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS \"db1\"");
+
+ session.executeNonQueryStatement("USE \"db1\"");
+ // only one column in this table, and others should be auto-created
+ session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS table1 (id1 string id)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id2", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id2", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id2", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ // id 1 should be null
+ assertNull(rowRecord.getFields().get(1).getDataType());
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001);
+ cnt++;
+ }
+ assertEquals(30, cnt);
+ }
+ // insertRelationalTabletPerformanceTest();
}
private static void insertRelationalTabletPerformanceTest()
@@ -162,8 +229,8 @@ public class IoTDBSessionRelationalIT {
for (long row = 0; row < 15; row++) {
session.executeNonQueryStatement(
String.format(
- "INSERT INTO table1 (id1, attr1, m1) VALUES ('%s', '%s', %f)",
- "id:" + row, "attr:" + row, row * 1.0));
+ "INSERT INTO table1 (time, id1, attr1, m1) VALUES (%d, '%s', '%s', %f)",
+ row, "id:" + row, "attr:" + row, row * 1.0));
}
session.executeNonQueryStatement("FLush");
@@ -171,18 +238,21 @@ public class IoTDBSessionRelationalIT {
for (long row = 15; row < 30; row++) {
session.executeNonQueryStatement(
String.format(
- "INSERT INTO table1 (id1, attr1, m1) VALUES ('%s', '%s', %f)",
- "id:" + row, "attr:" + row, row * 1.0));
+ "INSERT INTO table1 (time, id1, attr1, m1) VALUES (%d, '%s', '%s', %f)",
+ row, "id:" + row, "attr:" + row, row * 1.0));
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ cnt++;
}
+ assertEquals(30, cnt);
// sql cannot create column
assertThrows(
@@ -193,14 +263,6 @@ public class IoTDBSessionRelationalIT {
"INSERT INTO table1 (id1, id2, attr1, m1) VALUES ('%s', '%s', '%s', %f)",
"id:" + 100, "id:" + 100, "attr:" + 100, 100 * 1.0)));
}
- Map<String, ChunkMetadata> chunkMetadataMap = new HashMap<>();
- List<ChunkMetadata> valueChunkMetadataList = new ArrayList<>();
- chunkMetadataMap.computeIfPresent(
- "",
- (k, v) -> {
- valueChunkMetadataList.add(v);
- return v;
- });
}
@Test
@@ -269,6 +331,7 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
@@ -277,9 +340,9 @@ public class IoTDBSessionRelationalIT {
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
// "m2" should not be present
assertEquals(4, rowRecord.getFields().size());
- timestamp++;
- // System.out.println(rowRecord);
+ cnt++;
}
+ assertEquals(30, cnt);
} finally {
EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
}
@@ -326,13 +389,16 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
@@ -438,6 +504,7 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
@@ -446,9 +513,9 @@ public class IoTDBSessionRelationalIT {
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
// "m2" should not be present
assertEquals(4, rowRecord.getFields().size());
- timestamp++;
- // System.out.println(rowRecord);
+ cnt++;
}
+ assertEquals(30, cnt);
} finally {
EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
}
@@ -511,6 +578,7 @@ public class IoTDBSessionRelationalIT {
tablet.reset();
}
+ int cnt = 0;
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
@@ -518,7 +586,9 @@ public class IoTDBSessionRelationalIT {
assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
@@ -558,6 +628,7 @@ public class IoTDBSessionRelationalIT {
"table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
}
+ int cnt = 0;
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
@@ -565,7 +636,9 @@ public class IoTDBSessionRelationalIT {
assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
@@ -625,15 +698,16 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
- timestamp++;
- // System.out.println(rowRecord);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
@@ -693,6 +767,7 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
@@ -701,9 +776,9 @@ public class IoTDBSessionRelationalIT {
assertEquals("id:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals("attr:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001);
- timestamp++;
- // System.out.println(rowRecord);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
@@ -752,6 +827,7 @@ public class IoTDBSessionRelationalIT {
}
SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
timestamp = rowRecord.getFields().get(0).getLongV();
@@ -759,7 +835,9 @@ public class IoTDBSessionRelationalIT {
assertEquals("id2:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
assertEquals("attr1:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString());
assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001);
+ cnt++;
}
+ assertEquals(30, cnt);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
index b6faadf5ebb..3c51cd4d195 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
@@ -134,7 +134,7 @@ public class TableHeaderSchemaValidator {
// TODO table metadata: authority check for table alter
// check id or attribute column data type in this method
autoCreateColumn(database, tableSchema.getTableName(), missingColumnList, context);
- resultColumnList.addAll(missingColumnList);
+ table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName());
}
table
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
index a609d5f5218..b78830a1dab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
@@ -74,16 +74,21 @@ public abstract class WrappedInsertStatement extends WrappedStatement
List<ColumnSchema> columnSchemas =
new ArrayList<>(insertBaseStatement.getMeasurements().length);
for (int i = 0; i < insertBaseStatement.getMeasurements().length; i++) {
- columnSchemas.add(
- new ColumnSchema(
- insertBaseStatement.getMeasurements()[i],
- insertBaseStatement.getDataTypes() != null
- ? TypeFactory.getType(insertBaseStatement.getDataTypes()[i])
- : null,
- false,
- insertBaseStatement.getColumnCategories() != null
- ? insertBaseStatement.getColumnCategories()[i]
- : null));
+ if (insertBaseStatement.getMeasurements()[i] != null) {
+ columnSchemas.add(
+ new ColumnSchema(
+ insertBaseStatement.getMeasurements()[i],
+ insertBaseStatement.getDataTypes() != null
+ && insertBaseStatement.getDataTypes()[i] != null
+ ? TypeFactory.getType(insertBaseStatement.getDataTypes()[i])
+ : null,
+ false,
+ insertBaseStatement.getColumnCategories() != null
+ ? insertBaseStatement.getColumnCategories()[i]
+ : null));
+ } else {
+ columnSchemas.add(null);
+ }
}
return new TableSchema(tableName, columnSchemas);
}
@@ -122,7 +127,7 @@ public abstract class WrappedInsertStatement extends WrappedStatement
*/
public void adjustIdColumns(
List<ColumnSchema> realIdColumnSchemas, final InsertBaseStatement baseStatement) {
- List<ColumnSchema> incomingColumnSchemas = getTableSchema().getColumns();
+ List<ColumnSchema> incomingColumnSchemas = toTableSchema(baseStatement).getColumns();
for (int realIdColPos = 0; realIdColPos < realIdColumnSchemas.size(); realIdColPos++) {
ColumnSchema realColumn = realIdColumnSchemas.get(realIdColPos);
int incomingIdColPos = incomingColumnSchemas.indexOf(realColumn);