You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/07/22 00:43:17 UTC
(iotdb) branch ty/TableModelGrammar updated: Table model ingestion
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty/TableModelGrammar by this push:
new 36e49f2331d Table model ingestion
36e49f2331d is described below
commit 36e49f2331df7127bec2a514a9710b81a52d5887
Author: Jiang Tian <jt...@163.com>
AuthorDate: Mon Jul 22 08:43:12 2024 +0800
Table model ingestion
---
.../iotdb/session/it/IoTDBSessionRelationalIT.java | 765 +++++++++++++++++++++
.../java/org/apache/iotdb/isession/ISession.java | 16 +
.../java/org/apache/iotdb/session/Session.java | 108 +--
.../apache/iotdb/session/SessionConnection.java | 16 +-
.../iotdb/confignode/manager/ConfigManager.java | 85 ++-
.../apache/iotdb/confignode/manager/IManager.java | 17 +
.../persistence/schema/ClusterSchemaInfo.java | 44 +-
.../confignode/persistence/schema/ConfigMTree.java | 2 +-
.../impl/schema/table/AddTableColumnProcedure.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 14 +
.../dataregion/DataExecutionVisitor.java | 18 +
.../dataregion/DataRegionStateMachine.java | 1 +
.../db/exception/query/QueryProcessException.java | 5 +
.../iotdb/db/exception/sql/SemanticException.java | 9 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 1 +
.../visitor/PipePlanToStatementVisitor.java | 18 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 18 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 63 +-
.../db/queryengine/common/MPPQueryContext.java | 10 +
.../execution/executor/RegionWriteExecutor.java | 7 +
.../operator/process/CollectOperator.java | 3 +
.../schema/source/TableDeviceQuerySource.java | 3 +
.../iotdb/db/queryengine/plan/Coordinator.java | 57 +-
.../db/queryengine/plan/analyze/Analysis.java | 18 +-
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 271 ++++++++
.../queryengine/plan/analyze/AnalyzeVisitor.java | 305 +++-----
.../db/queryengine/plan/analyze/Analyzer.java | 4 +
.../plan/analyze/ClusterPartitionFetcher.java | 93 ++-
.../db/queryengine/plan/analyze/IAnalysis.java | 15 +
.../plan/analyze/LoadTsfileAnalyzer.java | 2 +-
.../queryengine/plan/analyze/SelectIntoUtils.java | 4 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 17 +-
.../plan/analyze/schema/ISchemaAutoCreation.java | 2 +
.../plan/analyze/schema/SchemaValidator.java | 26 +
.../execution/config/TableConfigTaskVisitor.java | 9 +
.../config/executor/ClusterConfigTaskExecutor.java | 1 +
.../relational/AlterTableAddColumnTask.java | 2 +
.../metadata/relational/CreateTableTask.java | 2 +
.../memory/StatementMemorySourceVisitor.java | 2 +-
.../plan/optimization/AggregationPushDown.java | 2 +-
.../plan/optimization/ColumnInjectionPushDown.java | 2 +-
.../plan/optimization/LimitOffsetPushDown.java | 2 +-
.../OrderByExpressionWithLimitChangeToTopK.java | 2 +-
.../plan/optimization/PredicatePushDown.java | 2 +-
.../plan/parser/StatementGenerator.java | 34 +
.../queryengine/plan/planner/LogicalPlanner.java | 3 +-
.../queryengine/plan/planner/TreeModelPlanner.java | 6 +-
.../planner/distribution/DistributionPlanner.java | 10 +-
.../SimpleFragmentParallelPlanner.java | 14 +-
.../distribution/WriteFragmentParallelPlanner.java | 19 +-
.../plan/planner/plan/node/PlanNodeType.java | 27 +-
.../plan/planner/plan/node/PlanVisitor.java | 15 +
.../planner/plan/node/load/LoadTsFileNode.java | 7 +-
.../node/metedata/read/TableDeviceQueryNode.java | 15 +-
.../plan/node/write/InsertMultiTabletsNode.java | 5 -
.../plan/planner/plan/node/write/InsertNode.java | 88 ++-
.../planner/plan/node/write/InsertRowNode.java | 67 +-
.../planner/plan/node/write/InsertRowsNode.java | 25 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 21 +-
.../planner/plan/node/write/InsertTabletNode.java | 354 +++++++---
.../plan/node/write/RelationalInsertRowNode.java | 227 ++++++
.../plan/node/write/RelationalInsertRowsNode.java | 188 +++++
.../node/write/RelationalInsertTabletNode.java | 247 +++++++
.../plan/relational/analyzer/Analysis.java | 45 +-
.../plan/relational/analyzer/Analyzer.java | 1 +
.../relational/analyzer/ExpressionAnalyzer.java | 1 +
.../relational/analyzer/StatementAnalyzer.java | 46 +-
.../analyzer/StatementAnalyzerFactory.java | 2 +-
.../plan/relational/metadata/ColumnSchema.java | 11 +-
.../plan/relational/metadata/Metadata.java | 12 +
.../relational/metadata/TableMetadataImpl.java | 6 +
.../plan/relational/metadata/TableSchema.java | 67 ++
.../fetcher/TableHeaderSchemaValidator.java | 13 +-
.../plan/relational/planner/LogicalPlanner.java | 44 +-
.../plan/relational/planner/RelationPlanner.java | 73 ++
.../distribute/DistributedPlanGenerator.java | 11 +-
.../distribute/TableDistributionPlanner.java | 6 +-
.../plan/relational/planner/node/CollectNode.java | 29 +-
.../planner/node/CreateTableDeviceNode.java | 5 +-
.../optimizations/PushPredicateIntoTableScan.java | 12 +
.../plan/relational/sql/ast/AstVisitor.java | 16 +
.../plan/relational/sql/ast/BinaryLiteral.java | 6 +
.../plan/relational/sql/ast/BooleanLiteral.java | 5 +
.../plan/relational/sql/ast/DecimalLiteral.java | 7 +
.../plan/relational/sql/ast/DoubleLiteral.java | 5 +
.../plan/relational/sql/ast/FetchDevice.java | 5 +-
.../sql/ast/{Literal.java => Flush.java} | 19 +-
.../plan/relational/sql/ast/GenericLiteral.java | 7 +
.../plan/relational/sql/ast/InsertRow.java | 91 +++
.../plan/relational/sql/ast/InsertRows.java | 143 ++++
.../plan/relational/sql/ast/InsertTablet.java | 105 +++
.../plan/relational/sql/ast/Literal.java | 5 +
.../plan/relational/sql/ast/LongLiteral.java | 5 +
.../plan/relational/sql/ast/NullLiteral.java | 5 +
.../plan/relational/sql/ast/StringLiteral.java | 7 +
.../relational/sql/ast/WrappedInsertStatement.java | 198 ++++++
.../plan/relational/sql/ast/WrappedStatement.java | 83 +++
.../plan/relational/sql/parser/AstBuilder.java | 111 ++-
.../plan/relational/sql/util/AstUtil.java | 10 +
.../plan/relational/utils/ConversionUtils.java} | 13 +-
.../db/queryengine/plan/statement/Statement.java | 6 +
.../plan/statement/crud/InsertBaseStatement.java | 141 ++++
.../plan/statement/crud/InsertRowStatement.java | 46 +-
.../plan/statement/crud/InsertRowsStatement.java | 6 +
.../plan/statement/crud/InsertTabletStatement.java | 95 +++
.../attribute/DeviceAttributeStore.java | 9 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 19 +-
.../db/schemaengine/table/DataNodeTableCache.java | 19 +-
.../db/storageengine/dataregion/DataRegion.java | 300 +++++---
.../dataregion/memtable/AbstractMemTable.java | 66 +-
.../memtable/AlignedWritableMemChunk.java | 12 +-
.../memtable/AlignedWritableMemChunkGroup.java | 6 +-
.../dataregion/memtable/IMemTable.java | 6 +-
.../dataregion/memtable/IWritableMemChunk.java | 6 +-
.../memtable/IWritableMemChunkGroup.java | 4 +-
.../dataregion/memtable/TsFileProcessor.java | 405 +++++++----
.../dataregion/memtable/WritableMemChunk.java | 6 +-
.../dataregion/memtable/WritableMemChunkGroup.java | 4 +-
.../wal/recover/file/TsFilePlanRedoer.java | 2 +-
.../db/trigger/executor/TriggerFireVisitor.java | 8 +
.../org/apache/iotdb/db/utils/CommonUtils.java | 77 +++
.../java/org/apache/iotdb/db/utils/MemUtils.java | 34 +-
.../db/utils/datastructure/AlignedTVList.java | 37 +-
.../iotdb/db/utils/datastructure/TVList.java | 4 +-
.../schemaRegion/SchemaRegionTestUtil.java | 17 +-
.../db/queryengine/plan/analyze/AnalyzeTest.java | 2 +-
.../plan/optimization/OptimizationTestUtil.java | 4 +-
.../plan/parser/StatementGeneratorTest.java | 188 +++++
.../distribution/DistributionPlannerBasicTest.java | 19 +-
.../plan/planner/distribution/Util.java | 5 +-
.../plan/planner/distribution/Util2.java | 4 +-
.../planner/logical/LogicalPlannerTestUtil.java | 2 +-
.../node/write/InsertRowsNodeSerdeTest.java | 161 ++++-
.../node/write/InsertTabletNodeSerdeTest.java | 158 +++++
.../planner/node/write/WritePlanNodeSplitTest.java | 5 +-
.../plan/relational/analyzer/AnalyzerTest.java | 207 +++++-
.../analyzer/MockTableModelDataPartition.java | 2 +-
.../plan/statement/InsertStatementTest.java | 182 +++++
.../plan/statement/StatementTestUtils.java | 257 +++++++
.../builder/EvaluationDAGBuilderTest.java | 7 +-
.../storageengine/dataregion/DataRegionTest.java | 110 +++
.../dataregion/memtable/PrimitiveMemTableTest.java | 2 +-
.../dataregion/memtable/TsFileProcessorTest.java | 32 +-
.../apache/iotdb/db/utils/DateTimeUtilsTest.java | 2 +
.../org/apache/iotdb/db/utils/MemUtilsTest.java | 13 +-
.../db/utils/datastructure/AlignedTVListTest.java | 8 +-
.../commons/exception/IoTDBRuntimeException.java | 66 ++
.../iotdb/commons/partition/DataPartition.java | 41 +-
.../commons/partition/DataPartitionQueryParam.java | 9 +
.../iotdb/commons/partition/SchemaPartition.java | 4 +-
.../apache/iotdb/commons/path/AlignedFullPath.java | 19 +
.../apache/iotdb/commons/schema/table/TsTable.java | 15 +-
.../schema/table/column/TsTableColumnCategory.java | 33 +-
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 18 +-
.../org/apache/iotdb/commons/utils/PathUtils.java | 7 +
.../src/main/thrift/confignode.thrift | 10 +
.../thrift-datanode/src/main/thrift/client.thrift | 4 +
157 files changed, 6590 insertions(+), 1008 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
new file mode 100644
index 00000000000..3a06c9fe433
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.session.it;
+
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+
+@RunWith(IoTDBTestRunner.class)
+public class IoTDBSessionRelationalIT {
+
+ @Before
+ public void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("CREATE DATABASE db1");
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("DROP DATABASE db1");
+ }
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ // for manual debugging
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException {
+ insertRelationalTabletPerformanceTest();
+ }
+
+ private static void insertRelationalTabletPerformanceTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session =
+ new Session.Builder().host("127.0.0.1").port(6667).sqlDialect(TABLE_SQL_DIALECT).build()) {
+ session.open();
+ session.executeNonQueryStatement("USE \"db1\"");
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertRelationalSqlTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ long timestamp;
+
+ for (long row = 0; row < 15; row++) {
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO table1 (id1, attr1, m1) VALUES ('%s', '%s', %f)",
+ "id:" + row, "attr:" + row, row * 1.0));
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO table1 (id1, attr1, m1) VALUES ('%s', '%s', %f)",
+ "id:" + row, "attr:" + row, row * 1.0));
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ }
+
+ // sql cannot create column
+ assertThrows(
+ StatementExecutionException.class,
+ () ->
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO table1 (id1, id2, attr1, m1) VALUES ('%s', '%s', '%s', %f)",
+ "id:" + 100, "id:" + 100, "attr:" + 100, 100 * 1.0)));
+ }
+ Map<String, ChunkMetadata> chunkMetadataMap = new HashMap<>();
+ List<ChunkMetadata> valueChunkMetadataList = new ArrayList<>();
+ chunkMetadataMap.computeIfPresent(
+ "",
+ (k, v) -> {
+ valueChunkMetadataList.add(v);
+ return v;
+ });
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void partialInsertRelationalRowTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ // disable auto-creation only for this test
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(false);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("CREATE DATABASE \"db1\"");
+ session.executeNonQueryStatement("USE \"db1\"");
+ // the table is missing column "m2"
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ // the insertion contains "m2"
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ schemaList.add(new MeasurementSchema("m2", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(
+ ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT, ColumnType.MEASUREMENT);
+ List<String> measurementIds =
+ schemaList.stream()
+ .map(IMeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ List<TSDataType> dataTypes =
+ schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+
+ long timestamp = 0;
+
+ for (long row = 0; row < 15; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0, row * 1.0};
+ try {
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ } catch (StatementExecutionException e) {
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0, row * 1.0};
+ try {
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ } catch (StatementExecutionException e) {
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ // "m2" should not be present
+ assertEquals(4, rowRecord.getFields().size());
+ timestamp++;
+ // System.out.println(rowRecord);
+ }
+ } finally {
+ EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertRelationalRowTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+ List<String> measurementIds =
+ schemaList.stream()
+ .map(IMeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ List<TSDataType> dataTypes =
+ schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+
+ long timestamp = 0;
+
+ for (long row = 0; row < 15; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void partialInsertRelationalTabletTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ // disable auto-creation only for this test
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(false);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("CREATE DATABASE \"db1\"");
+ session.executeNonQueryStatement("USE \"db1\"");
+ // the table is missing column "m2"
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ // the insertion contains "m2"
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ schemaList.add(new MeasurementSchema("m2", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(
+ ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ tablet.addValue("m2", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ try {
+ session.insertRelationalTablet(tablet, true);
+ } catch (StatementExecutionException e) {
+ // a partial insertion should be reported
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ try {
+ session.insertRelationalTablet(tablet, true);
+ } catch (StatementExecutionException e) {
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ tablet.addValue("m2", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ try {
+ session.insertRelationalTablet(tablet, true);
+ } catch (StatementExecutionException e) {
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ try {
+ session.insertRelationalTablet(tablet, true);
+ } catch (StatementExecutionException e) {
+ if (!e.getMessage()
+ .equals(
+ "507: Fail to insert measurements [m2] caused by [Column m2 does not exists or fails to be created]")) {
+ throw e;
+ }
+ }
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ // "m2" should not be present
+ assertEquals(4, rowRecord.getFields().size());
+ timestamp++;
+ // System.out.println(rowRecord);
+ }
+ } finally {
+ EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertRelationalTabletTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void autoCreateTableTest() throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // no table created here
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+ List<String> measurementIds =
+ schemaList.stream()
+ .map(IMeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ List<TSDataType> dataTypes =
+ schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+
+ long timestamp = 0;
+
+ for (long row = 0; row < 15; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void autoCreateNonIdColumnTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // only one column in this table, and others should be auto-created
+ session.executeNonQueryStatement("CREATE TABLE table1 (id1 string id)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id1", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001);
+ timestamp++;
+ // System.out.println(rowRecord);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void autoCreateIdColumnTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // only one column in this table, and others should be auto-created
+ session.executeNonQueryStatement("CREATE TABLE table1 (id1 string id)");
+
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id2", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+
+ long timestamp = 0;
+ Tablet tablet = new Tablet("table1", schemaList, columnTypes, 15);
+
+ for (long row = 0; row < 15; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id2", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, timestamp + row);
+ tablet.addValue("id2", rowIndex, "id:" + row);
+ tablet.addValue("attr1", rowIndex, "attr:" + row);
+ tablet.addValue("m1", rowIndex, row * 1.0);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertRelationalTablet(tablet, true);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertRelationalTablet(tablet);
+ tablet.reset();
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ // id 1 should be null
+ assertNull(rowRecord.getFields().get(1).getDataType());
+ assertEquals("id:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals("attr:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001);
+ timestamp++;
+ // System.out.println(rowRecord);
+ }
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void autoAdjustIdTest() throws IoTDBConnectionException, StatementExecutionException {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) {
+ session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
+ session.executeNonQueryStatement("CREATE DATABASE db1");
+ session.executeNonQueryStatement("USE \"db1\"");
+ // the id order in the table is (id1, id2)
+ session.executeNonQueryStatement(
+ "CREATE TABLE table1 (id1 string id, id2 string id, attr1 string attribute, "
+ + "m1 double "
+ + "measurement)");
+
+ // the id order in the row is (id2, id1)
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id2", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE));
+ final List<ColumnType> columnTypes =
+ Arrays.asList(ColumnType.ID, ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT);
+ List<String> measurementIds =
+ schemaList.stream()
+ .map(IMeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ List<TSDataType> dataTypes =
+ schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+
+ long timestamp = 0;
+
+ for (long row = 0; row < 15; row++) {
+ Object[] values = new Object[] {"id2:" + row, "id1:" + row, "attr1:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ session.executeNonQueryStatement("FLush");
+
+ for (long row = 15; row < 30; row++) {
+ Object[] values = new Object[] {"id2:" + row, "id1:" + row, "attr1:" + row, row * 1.0};
+ session.insertRelationalRecord(
+ "table1", timestamp + row, measurementIds, dataTypes, columnTypes, values);
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("select * from table1 order by time");
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ timestamp = rowRecord.getFields().get(0).getLongV();
+ assertEquals("id1:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString());
+ assertEquals("id2:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString());
+ assertEquals("attr1:" + timestamp, rowRecord.getFields().get(3).getBinaryV().toString());
+ assertEquals(timestamp * 1.0, rowRecord.getFields().get(4).getDoubleV(), 0.0001);
+ }
+ }
+ }
+}
diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java
index 85a8c9c00d5..7bc31731802 100644
--- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java
+++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java
@@ -34,6 +34,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -212,6 +213,15 @@ public interface ISession extends AutoCloseable {
Object... values)
throws IoTDBConnectionException, StatementExecutionException;
+ void insertRelationalRecord(
+ String tableName,
+ long time,
+ List<String> measurements,
+ List<TSDataType> types,
+ List<ColumnType> columnCategories,
+ Object... values)
+ throws IoTDBConnectionException, StatementExecutionException;
+
void insertRecord(
String deviceId,
long time,
@@ -336,6 +346,12 @@ public interface ISession extends AutoCloseable {
void insertTablet(Tablet tablet, boolean sorted)
throws IoTDBConnectionException, StatementExecutionException;
+ void insertRelationalTablet(Tablet tablet, boolean sorted)
+ throws IoTDBConnectionException, StatementExecutionException;
+
+ void insertRelationalTablet(Tablet tablet)
+ throws IoTDBConnectionException, StatementExecutionException;
+
void insertAlignedTablet(Tablet tablet)
throws StatementExecutionException, IoTDBConnectionException;
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
index e80915d2598..cf4095a9036 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -70,6 +70,7 @@ import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
@@ -1226,6 +1227,38 @@ public class Session implements ISession {
insertRecord(deviceId, request);
}
+ /**
+ * insert data in one row to the table model, if you want to improve your performance, please use
+ * insertRecords method or insertTablet method
+ *
+ * @see Session#insertRecords(List, List, List, List, List)
+ * @see Session#insertTablet(Tablet)
+ */
+ @Override
+ public void insertRelationalRecord(
+ String tableName,
+ long time,
+ List<String> measurements,
+ List<TSDataType> types,
+ List<ColumnType> columnCategories,
+ Object... values)
+ throws IoTDBConnectionException, StatementExecutionException {
+ TSInsertRecordReq request;
+ try {
+ request =
+ filterAndGenTSInsertRecordReq(
+ tableName, time, measurements, types, Arrays.asList(values), false);
+ request.setColumnCategoryies(
+ columnCategories.stream().map(c -> (byte) c.ordinal()).collect(Collectors.toList()));
+ request.setIsWriteToTable(true);
+ } catch (NoValidValueException e) {
+ logger.warn(ALL_VALUES_ARE_NULL, tableName, time, measurements);
+ return;
+ }
+
+ insertRecord(tableName, request);
+ }
+
private void insertRecord(String prefixPath, TSInsertRecordReq request)
throws IoTDBConnectionException, StatementExecutionException {
try {
@@ -1579,14 +1612,7 @@ public class Session implements ISession {
}
}
- /**
- * When the value is null,filter this,don't use this measurement.
- *
- * @param times
- * @param measurementsList
- * @param valuesList
- * @param typesList
- */
+ /** When the value is null,filter this,don't use this measurement. */
private void filterNullValueAndMeasurement(
List<String> deviceIds,
List<Long> times,
@@ -1612,15 +1638,7 @@ public class Session implements ISession {
}
}
- /**
- * Filter the null value of list。
- *
- * @param deviceId
- * @param times
- * @param measurementsList
- * @param typesList
- * @param valuesList
- */
+ /** Filter the null value of list。 */
private void filterNullValueAndMeasurementOfOneDevice(
String deviceId,
List<Long> times,
@@ -1645,14 +1663,7 @@ public class Session implements ISession {
}
}
- /**
- * Filter the null value of list。
- *
- * @param times
- * @param deviceId
- * @param measurementsList
- * @param valuesList
- */
+ /** Filter the null value of list。 */
private void filterNullValueAndMeasurementWithStringTypeOfOneDevice(
List<Long> times,
String deviceId,
@@ -1677,10 +1688,6 @@ public class Session implements ISession {
/**
* Filter the null object of list。
*
- * @param deviceId
- * @param measurementsList
- * @param types
- * @param valuesList
* @return true:all value is null;false:not all null value is null.
*/
private boolean filterNullValueAndMeasurement(
@@ -1710,9 +1717,6 @@ public class Session implements ISession {
* Filter the null object of list。
*
* @param prefixPaths devices path。
- * @param times
- * @param measurementsList
- * @param valuesList
* @return true:all values of valuesList are null;false:Not all values of valuesList are null.
*/
private void filterNullValueAndMeasurementWithStringType(
@@ -1740,8 +1744,6 @@ public class Session implements ISession {
/**
* When the value is null,filter this,don't use this measurement.
*
- * @param valuesList
- * @param measurementsList
* @return true:all value is null;false:not all null value is null.
*/
private boolean filterNullValueAndMeasurementWithStringType(
@@ -2639,6 +2641,11 @@ public class Session implements ISession {
public void insertTablet(Tablet tablet, boolean sorted)
throws IoTDBConnectionException, StatementExecutionException {
TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, false);
+ insertTabletInternal(tablet, request);
+ }
+
+ private void insertTabletInternal(Tablet tablet, TSInsertTabletReq request)
+ throws IoTDBConnectionException, StatementExecutionException {
try {
getSessionConnection(tablet.getDeviceId()).insertTablet(request);
} catch (RedirectException e) {
@@ -2661,6 +2668,33 @@ public class Session implements ISession {
}
}
+ /**
+ * insert a relational Tablet
+ *
+ * @param tablet data batch
+ * @param sorted deprecated, whether times in Tablet are in ascending order
+ */
+ @Override
+ public void insertRelationalTablet(Tablet tablet, boolean sorted)
+ throws IoTDBConnectionException, StatementExecutionException {
+ TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, false);
+ request.setWriteToTable(true);
+ request.setColumnCategories(
+ tablet.getColumnTypes().stream().map(t -> (byte) t.ordinal()).collect(Collectors.toList()));
+ insertTabletInternal(tablet, request);
+ }
+
+ /**
+ * insert a relational Tablet
+ *
+ * @param tablet data batch
+ */
+ @Override
+ public void insertRelationalTablet(Tablet tablet)
+ throws IoTDBConnectionException, StatementExecutionException {
+ insertRelationalTablet(tablet, false);
+ }
+
/**
* insert the aligned timeseries data of a device. For each timestamp, the number of measurements
* is the same.
@@ -3470,8 +3504,6 @@ public class Session implements ISession {
* @param encodings the encoding of each measurement, only the first one in each nested list
* matters as above
* @param compressors the compressor of each measurement
- * @throws IoTDBConnectionException
- * @throws StatementExecutionException
* @deprecated
*/
@Override
@@ -3777,9 +3809,6 @@ public class Session implements ISession {
* <li>{@link TSInsertStringRecordsReq}
* <li>{@link TSInsertTabletsReq}
* </ul>
- *
- * @throws IoTDBConnectionException
- * @throws StatementExecutionException
*/
@SuppressWarnings({
"squid:S3776"
@@ -3875,6 +3904,7 @@ public class Session implements ISession {
}
public static class Builder {
+
private String host = SessionConfig.DEFAULT_HOST;
private int rpcPort = SessionConfig.DEFAULT_PORT;
private String username = SessionConfig.DEFAULT_USER;
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
index 3df2f1a351c..d2d5e7ebc2e 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java
@@ -1088,9 +1088,8 @@ public class SessionConnection {
return client.insertStringRecordsOfOneDevice(request);
}
- protected void insertTablet(TSInsertTabletReq request)
- throws IoTDBConnectionException, StatementExecutionException, RedirectException {
-
+ protected void withRetry(TFunction<TSStatus> function)
+ throws StatementExecutionException, RedirectException, IoTDBConnectionException {
TException lastTException = null;
TSStatus status = null;
for (int i = 0; i <= maxRetryCount; i++) {
@@ -1110,7 +1109,7 @@ public class SessionConnection {
}
}
try {
- status = insertTabletInternal(request);
+ status = function.run();
// need retry
if (status.isSetNeedRetry() && status.isNeedRetry()) {
continue;
@@ -1139,6 +1138,11 @@ public class SessionConnection {
}
}
+ protected void insertTablet(TSInsertTabletReq request)
+ throws IoTDBConnectionException, StatementExecutionException, RedirectException {
+ withRetry(() -> insertTabletInternal(request));
+ }
+
private TSStatus insertTabletInternal(TSInsertTabletReq request) throws TException {
request.setSessionId(sessionId);
return client.insertTablet(request);
@@ -1686,4 +1690,8 @@ public class SessionConnection {
public String toString() {
return "SessionConnection{" + " endPoint=" + endPoint + "}";
}
+
+ private interface TFunction<T> {
+ T run() throws TException;
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index eb08141359d..04725887121 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -702,10 +702,10 @@ public class ConfigManager implements IManager {
@Override
public TSchemaPartitionTableResp getSchemaPartition(PathPatternTree patternTree) {
// Construct empty response
- TSchemaPartitionTableResp resp = new TSchemaPartitionTableResp();
TSStatus status = confirmLeader();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ TSchemaPartitionTableResp resp = new TSchemaPartitionTableResp();
return resp.setStatus(status);
}
@@ -738,30 +738,40 @@ public class ConfigManager implements IManager {
}
}
+ Map<String, List<TSeriesPartitionSlot>> databaseSlotMap = new HashMap<>();
+ partitionSlotsMap.forEach((db, slots) -> databaseSlotMap.put(db, new ArrayList<>(slots)));
+ return getSchemaPartition(databaseSlotMap);
+ }
+
+ @Override
+ public TSchemaPartitionTableResp getSchemaPartition(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) {
+ // Construct empty response
+ TSchemaPartitionTableResp resp = new TSchemaPartitionTableResp();
// Return empty resp if the partitionSlotsMap is empty
- if (partitionSlotsMap.isEmpty()) {
+ if (dbSlotMap.isEmpty()) {
return resp.setStatus(StatusUtils.OK).setSchemaPartitionTable(new HashMap<>());
}
GetSchemaPartitionPlan getSchemaPartitionPlan =
new GetSchemaPartitionPlan(
- partitionSlotsMap.entrySet().stream()
+ dbSlotMap.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> new ArrayList<>(e.getValue()))));
SchemaPartitionResp queryResult = partitionManager.getSchemaPartition(getSchemaPartitionPlan);
resp = queryResult.convertToRpcSchemaPartitionTableResp();
- LOGGER.debug("GetSchemaPartition receive paths: {}, return: {}", relatedPaths, resp);
+ LOGGER.debug("GetSchemaPartition receive paths: {}, return: {}", dbSlotMap, resp);
return resp;
}
@Override
public TSchemaPartitionTableResp getOrCreateSchemaPartition(PathPatternTree patternTree) {
- // Construct empty response
- TSchemaPartitionTableResp resp = new TSchemaPartitionTableResp();
TSStatus status = confirmLeader();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ // Construct empty response
+ TSchemaPartitionTableResp resp = new TSchemaPartitionTableResp();
return resp.setStatus(status);
}
@@ -769,40 +779,44 @@ public class ConfigManager implements IManager {
List<String> databases = getClusterSchemaManager().getDatabaseNames();
// Build GetOrCreateSchemaPartitionPlan
- Map<String, List<TSeriesPartitionSlot>> partitionSlotsMap = new HashMap<>();
+ Map<String, Set<TSeriesPartitionSlot>> partitionSlotsMap = new HashMap<>();
for (IDeviceID deviceID : devicePaths) {
for (String database : databases) {
if (PathUtils.isStartWith(deviceID, database)) {
partitionSlotsMap
- .computeIfAbsent(database, key -> new ArrayList<>())
+ .computeIfAbsent(database, key -> new HashSet<>())
.add(getPartitionManager().getSeriesPartitionSlot(deviceID));
break;
}
}
}
+
+ Map<String, List<TSeriesPartitionSlot>> partitionSlotListMap = new HashMap<>();
+ partitionSlotsMap.forEach((db, slots) -> partitionSlotListMap.put(db, new ArrayList<>(slots)));
+ return getOrCreateSchemaPartition(partitionSlotListMap);
+ }
+
+ @Override
+ public TSchemaPartitionTableResp getOrCreateSchemaPartition(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) {
+ // Construct empty response
+ TSchemaPartitionTableResp resp;
GetOrCreateSchemaPartitionPlan getOrCreateSchemaPartitionPlan =
- new GetOrCreateSchemaPartitionPlan(partitionSlotsMap);
+ new GetOrCreateSchemaPartitionPlan(dbSlotMap);
SchemaPartitionResp queryResult =
partitionManager.getOrCreateSchemaPartition(getOrCreateSchemaPartitionPlan);
resp = queryResult.convertToRpcSchemaPartitionTableResp();
if (CONF.isEnablePrintingNewlyCreatedPartition()) {
- printNewCreatedSchemaPartition(devicePaths, resp);
+ printNewCreatedSchemaPartition(dbSlotMap, resp);
}
return resp;
}
- private void printNewCreatedSchemaPartition(
- List<IDeviceID> deviceIDS, TSchemaPartitionTableResp resp) {
+ private String partitionTableRespToString(TSchemaPartitionTableResp resp) {
final String lineSeparator = System.lineSeparator();
- StringBuilder devicePathString = new StringBuilder("{");
- for (IDeviceID deviceID : deviceIDS) {
- devicePathString.append(lineSeparator).append("\t").append(deviceID).append(",");
- }
- devicePathString.append(lineSeparator).append("}");
-
StringBuilder schemaPartitionRespString = new StringBuilder("{");
schemaPartitionRespString
.append(lineSeparator)
@@ -832,11 +846,36 @@ public class ConfigManager implements IManager {
schemaPartitionRespString.append(lineSeparator).append("\t},");
}
schemaPartitionRespString.append(lineSeparator).append("}");
- LOGGER.debug(
- "[GetOrCreateSchemaPartition]:{}Receive PathPatternTree: {}, Return TSchemaPartitionTableResp: {}",
- lineSeparator,
- devicePathString,
- schemaPartitionRespString);
+ return schemaPartitionRespString.toString();
+ }
+
+ private void printNewCreatedSchemaPartition(
+ Map<String, List<TSeriesPartitionSlot>> databaseNameSlotMap, TSchemaPartitionTableResp resp) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "[GetOrCreateSchemaPartition]:{}Receive databaseNameSlotMap: {}, Return TSchemaPartitionTableResp: {}",
+ System.lineSeparator(),
+ databaseNameSlotMap,
+ partitionTableRespToString(resp));
+ }
+ }
+
+ private void printNewCreatedSchemaPartition(
+ List<IDeviceID> deviceIDS, TSchemaPartitionTableResp resp) {
+ final String lineSeparator = System.lineSeparator();
+ StringBuilder devicePathString = new StringBuilder("{");
+ for (IDeviceID deviceID : deviceIDS) {
+ devicePathString.append(lineSeparator).append("\t").append(deviceID).append(",");
+ }
+ devicePathString.append(lineSeparator).append("}");
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "[GetOrCreateSchemaPartition]:{}Receive PathPatternTree: {}, Return TSchemaPartitionTableResp: {}",
+ lineSeparator,
+ devicePathString,
+ partitionTableRespToString(resp));
+ }
}
@Override
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index 6aae4419cba..5e9dd2f57c2 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq;
import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq;
import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp;
@@ -130,6 +131,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import java.nio.ByteBuffer;
import java.util.List;
+import java.util.Map;
/**
* A subset of services provided by {@link ConfigManager}. For use internally only, passed to
@@ -358,6 +360,13 @@ public interface IManager {
*/
TSchemaPartitionTableResp getSchemaPartition(PathPatternTree patternTree);
+ /**
+ * Get SchemaPartition with <databaseName, seriesSlot>.
+ *
+ * @return TSchemaPartitionResp
+ */
+ TSchemaPartitionTableResp getSchemaPartition(Map<String, List<TSeriesPartitionSlot>> dbSlotMap);
+
/**
* Get or create SchemaPartition.
*
@@ -365,6 +374,14 @@ public interface IManager {
*/
TSchemaPartitionTableResp getOrCreateSchemaPartition(PathPatternTree patternTree);
+ /**
+ * Get or create SchemaPartition with <databaseName, seriesSlot>.
+ *
+ * @return TSchemaPartitionResp
+ */
+ TSchemaPartitionTableResp getOrCreateSchemaPartition(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap);
+
/**
* Create SchemaNodeManagementPartition for child paths node management.
*
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
index 103d88db0d6..02c97416698 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan;
@@ -101,7 +102,6 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE;
import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN;
import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE;
import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_TEMPLATE;
-import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT;
import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE_PATTERN;
import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY;
@@ -338,8 +338,9 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
public TSStatus setSchemaReplicationFactor(SetSchemaReplicationFactorPlan plan) {
TSStatus result = new TSStatus();
databaseReadWriteLock.writeLock().lock();
+ String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase());
try {
- PartialPath path = new PartialPath(plan.getDatabase());
+ PartialPath path = new PartialPath(databaseName);
if (mTree.isDatabaseAlreadySet(path)) {
mTree
.getDatabaseNodeByDatabasePath(path)
@@ -362,8 +363,9 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
public TSStatus setDataReplicationFactor(SetDataReplicationFactorPlan plan) {
TSStatus result = new TSStatus();
databaseReadWriteLock.writeLock().lock();
+ String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase());
try {
- PartialPath path = new PartialPath(plan.getDatabase());
+ PartialPath path = new PartialPath(databaseName);
if (mTree.isDatabaseAlreadySet(path)) {
mTree
.getDatabaseNodeByDatabasePath(path)
@@ -386,8 +388,9 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
public TSStatus setTimePartitionInterval(SetTimePartitionIntervalPlan plan) {
TSStatus result = new TSStatus();
databaseReadWriteLock.writeLock().lock();
+ String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase());
try {
- PartialPath path = new PartialPath(plan.getDatabase());
+ PartialPath path = new PartialPath(databaseName);
if (mTree.isDatabaseAlreadySet(path)) {
mTree
.getDatabaseNodeByDatabasePath(path)
@@ -1033,11 +1036,10 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
// region table management
public TSStatus preCreateTable(PreCreateTablePlan preCreateTablePlan) {
+ String databaseName = PathUtils.qualifyDatabaseName(preCreateTablePlan.getDatabase());
databaseReadWriteLock.writeLock().lock();
try {
- mTree.preCreateTable(
- new PartialPath(new String[] {ROOT, preCreateTablePlan.getDatabase()}),
- preCreateTablePlan.getTable());
+ mTree.preCreateTable(new PartialPath(databaseName), preCreateTablePlan.getTable());
return RpcUtils.SUCCESS_STATUS;
} catch (MetadataException e) {
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
@@ -1047,11 +1049,11 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
}
public TSStatus rollbackCreateTable(RollbackCreateTablePlan rollbackCreateTablePlan) {
+ String databaseName = PathUtils.qualifyDatabaseName(rollbackCreateTablePlan.getDatabase());
databaseReadWriteLock.writeLock().lock();
try {
mTree.rollbackCreateTable(
- new PartialPath(new String[] {ROOT, rollbackCreateTablePlan.getDatabase()}),
- rollbackCreateTablePlan.getTableName());
+ new PartialPath(databaseName), rollbackCreateTablePlan.getTableName());
return RpcUtils.SUCCESS_STATUS;
} catch (MetadataException e) {
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
@@ -1061,11 +1063,10 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
}
public TSStatus commitCreateTable(CommitCreateTablePlan commitCreateTablePlan) {
+ String databaseName = PathUtils.qualifyDatabaseName(commitCreateTablePlan.getDatabase());
databaseReadWriteLock.writeLock().lock();
try {
- mTree.commitCreateTable(
- new PartialPath(new String[] {ROOT, commitCreateTablePlan.getDatabase()}),
- commitCreateTablePlan.getTableName());
+ mTree.commitCreateTable(new PartialPath(databaseName), commitCreateTablePlan.getTableName());
return RpcUtils.SUCCESS_STATUS;
} catch (MetadataException e) {
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
@@ -1074,15 +1075,13 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
}
}
- public TShowTableResp showTables(final String database) {
+ public TShowTableResp showTables(String database) {
+ database = PathUtils.qualifyDatabaseName(database);
databaseReadWriteLock.readLock().lock();
try {
return new TShowTableResp(StatusUtils.OK)
.setTableInfoList(
- mTree
- .getAllUsingTablesUnderSpecificDatabase(
- new PartialPath(new String[] {ROOT, database}))
- .stream()
+ mTree.getAllUsingTablesUnderSpecificDatabase(new PartialPath(database)).stream()
.map(
tsTable ->
new TTableInfo(
@@ -1122,7 +1121,7 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
public TsTable getTsTable(String database, String tableName) {
databaseReadWriteLock.readLock().lock();
try {
- return mTree.getTable(new PartialPath(new String[] {ROOT, database}), tableName);
+ return mTree.getTable(new PartialPath(database), tableName);
} catch (MetadataException e) {
LOGGER.warn(e.getMessage(), e);
throw new RuntimeException(e);
@@ -1132,18 +1131,15 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
}
public TSStatus addTableColumn(AddTableColumnPlan plan) {
+ String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase());
databaseReadWriteLock.writeLock().lock();
try {
if (plan.isRollback()) {
mTree.rollbackAddTableColumn(
- new PartialPath(new String[] {ROOT, plan.getDatabase()}),
- plan.getTableName(),
- plan.getColumnSchemaList());
+ new PartialPath(databaseName), plan.getTableName(), plan.getColumnSchemaList());
} else {
mTree.addTableColumn(
- new PartialPath(new String[] {ROOT, plan.getDatabase()}),
- plan.getTableName(),
- plan.getColumnSchemaList());
+ new PartialPath(databaseName), plan.getTableName(), plan.getColumnSchemaList());
}
return RpcUtils.SUCCESS_STATUS;
} catch (MetadataException e) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java
index 00190fde455..bfa0adb60fd 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java
@@ -687,7 +687,7 @@ public class ConfigMTree {
return getAllDatabasePaths().stream()
.collect(
Collectors.toMap(
- databasePath -> databasePath.getFullPath().substring(ROOT.length() + 1),
+ PartialPath::getFullPath,
databasePath -> {
try {
return getAllUsingTablesUnderSpecificDatabase(databasePath);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
index b6017c1ce25..ab4e4693276 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
@@ -282,7 +282,7 @@ public class AddTableColumnProcedure
@Override
protected AddTableColumnState getInitialState() {
- return AddTableColumnState.ADD_COLUMN;
+ return AddTableColumnState.COLUMN_CHECK;
}
public String getDatabase() {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 5aedb696879..5597610980f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.common.rpc.thrift.TNodeLocations;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq;
import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq;
import org.apache.iotdb.common.rpc.thrift.TSetTTLReq;
@@ -191,6 +192,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
/** ConfigNodeRPCServer exposes the interface that interacts with the DataNode */
@@ -542,6 +544,12 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.getSchemaPartition(patternTree);
}
+ @Override
+ public TSchemaPartitionTableResp getSchemaPartitionTableWithSlots(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) {
+ return configManager.getSchemaPartition(dbSlotMap);
+ }
+
@Override
public TSchemaPartitionTableResp getOrCreateSchemaPartitionTable(TSchemaPartitionReq req) {
PathPatternTree patternTree =
@@ -549,6 +557,12 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.getOrCreateSchemaPartition(patternTree);
}
+ @Override
+ public TSchemaPartitionTableResp getOrCreateSchemaPartitionTableWithSlots(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) {
+ return configManager.getOrCreateSchemaPartition(dbSlotMap);
+ }
+
@Override
public TSchemaNodeManagementResp getSchemaNodeManagementPartition(TSchemaNodeManagementReq req) {
PathPatternTree patternTree =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
index 14b2229ed78..20c04f4c645 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
@@ -38,6 +38,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -56,6 +59,15 @@ public class DataExecutionVisitor extends PlanVisitor<TSStatus, DataRegion> {
return null;
}
+ public TSStatus visitRelationalInsertRow(RelationalInsertRowNode node, DataRegion context) {
+ return visitInsertRow(node, context);
+ }
+
+ @Override
+ public TSStatus visitRelationalInsertRows(RelationalInsertRowsNode node, DataRegion context) {
+ return visitInsertRows(node, context);
+ }
+
@Override
public TSStatus visitInsertRow(InsertRowNode node, DataRegion dataRegion) {
try {
@@ -73,6 +85,12 @@ public class DataExecutionVisitor extends PlanVisitor<TSStatus, DataRegion> {
}
}
+ @Override
+ public TSStatus visitRelationalInsertTablet(
+ RelationalInsertTabletNode node, DataRegion dataRegion) {
+ return visitInsertTablet(node, dataRegion);
+ }
+
@Override
public TSStatus visitInsertTablet(InsertTabletNode node, DataRegion dataRegion) {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
index f60ae0bf9a4..865b3edb4fd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
@@ -186,6 +186,7 @@ public class DataRegionStateMachine extends BaseStateMachine {
List<Integer> index = new ArrayList<>();
int i = 0;
switch (insertNodes.get(0).getType()) {
+ case RELATIONAL_INSERT_TABLET:
case INSERT_TABLET:
// merge to InsertMultiTabletsNode
List<InsertTabletNode> insertTabletNodes = new ArrayList<>(size);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
index c7460f381d0..08b6a61d06a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.exception.query;
import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import org.apache.iotdb.rpc.TSStatusCode;
public class QueryProcessException extends IoTDBException {
@@ -37,4 +38,8 @@ public class QueryProcessException extends IoTDBException {
public QueryProcessException(IoTDBException e) {
super(e, e.getErrorCode(), e.isUserException());
}
+
+ public QueryProcessException(IoTDBRuntimeException e) {
+ super(e, e.getErrorCode(), e.isUserException());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java
index 2e17d0775d9..66f3af08e91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java
@@ -19,13 +19,16 @@
package org.apache.iotdb.db.exception.sql;
-public class SemanticException extends RuntimeException {
+import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+public class SemanticException extends IoTDBRuntimeException {
public SemanticException(String message) {
- super(message);
+ super(message, TSStatusCode.SEMANTIC_ERROR.getStatusCode());
}
public SemanticException(Throwable cause) {
- super(cause);
+ super(cause, TSStatusCode.SEMANTIC_ERROR.getStatusCode());
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index d574d1a31fa..45f8b70a825 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -326,6 +326,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
switch (node.getType()) {
case INSERT_ROW:
case INSERT_TABLET:
+ case RELATIONAL_INSERT_TABLET:
dataContainers.add(
new TabletInsertionDataContainer(pipeTaskMeta, this, node, pipePattern));
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java
index 593e3715c88..1e9055b8ed9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
@@ -86,19 +87,14 @@ public class PipePlanToStatementVisitor extends PlanVisitor<Statement, Void> {
return statement;
}
+ @Override
+ public Statement visitRelationalInsertTablet(RelationalInsertTabletNode node, Void context) {
+ return new InsertTabletStatement(node);
+ }
+
@Override
public InsertTabletStatement visitInsertTablet(final InsertTabletNode node, final Void context) {
- final InsertTabletStatement statement = new InsertTabletStatement();
- statement.setDevicePath(node.getDevicePath());
- statement.setMeasurements(node.getMeasurements());
- statement.setTimes(node.getTimes());
- statement.setColumns(node.getColumns());
- statement.setBitMaps(node.getBitMaps());
- statement.setRowCount(node.getRowCount());
- statement.setDataTypes(node.getDataTypes());
- statement.setAligned(node.isAligned());
- statement.setMeasurementSchemas(node.getMeasurementSchemas());
- return statement;
+ return new InsertTabletStatement(node);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index 7e10700bfc0..351ed85ee4e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.common.rpc.thrift.TNodeLocations;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq;
import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq;
import org.apache.iotdb.common.rpc.thrift.TSetTTLReq;
@@ -162,6 +163,7 @@ import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.function.Predicate;
@@ -539,6 +541,14 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie
() -> client.getSchemaPartitionTable(req), resp -> !updateConfigNodeLeader(resp.status));
}
+ @Override
+ public TSchemaPartitionTableResp getSchemaPartitionTableWithSlots(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) throws TException {
+ return executeRemoteCallWithRetry(
+ () -> client.getSchemaPartitionTableWithSlots(dbSlotMap),
+ resp -> !updateConfigNodeLeader(resp.status));
+ }
+
@Override
public TSchemaPartitionTableResp getOrCreateSchemaPartitionTable(TSchemaPartitionReq req)
throws TException {
@@ -547,6 +557,14 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie
resp -> !updateConfigNodeLeader(resp.status));
}
+ @Override
+ public TSchemaPartitionTableResp getOrCreateSchemaPartitionTableWithSlots(
+ Map<String, List<TSeriesPartitionSlot>> dbSlotMap) throws TException {
+ return executeRemoteCallWithRetry(
+ () -> client.getOrCreateSchemaPartitionTableWithSlots(dbSlotMap),
+ resp -> !updateConfigNodeLeader(resp.status));
+ }
+
@Override
public TSchemaNodeManagementResp getSchemaNodeManagementPartition(TSchemaNodeManagementReq req)
throws TException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index 13d6e9d4216..83cf79a5c68 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -261,6 +261,7 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
@FunctionalInterface
public interface SelectResult {
+
boolean apply(TSExecuteStatementResp resp, IQueryExecution queryExecution, int fetchSize)
throws IoTDBException, IOException;
}
@@ -2013,15 +2014,28 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
// Step 2: call the coordinator
long queryId = SESSION_MANAGER.requestQueryId();
- ExecutionResult result =
- COORDINATOR.executeForTreeModel(
- statement,
- queryId,
- SESSION_MANAGER.getSessionInfo(clientSession),
- "",
- partitionFetcher,
- schemaFetcher);
-
+ ExecutionResult result;
+ if (statement.isWriteToTable()) {
+ result =
+ COORDINATOR.executeForTableModel(
+ statement,
+ relationSqlParser,
+ clientSession,
+ queryId,
+ SESSION_MANAGER.getSessionInfo(clientSession),
+ "",
+ metadata,
+ config.getConnectionTimeoutInMS());
+ } else {
+ result =
+ COORDINATOR.executeForTreeModel(
+ statement,
+ queryId,
+ SESSION_MANAGER.getSessionInfo(clientSession),
+ "",
+ partitionFetcher,
+ schemaFetcher);
+ }
return result.status;
} catch (IoTDBException e) {
return onIoTDBException(e, OperationType.INSERT_RECORD, e.getErrorCode());
@@ -2128,15 +2142,28 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
// Step 2: call the coordinator
long queryId = SESSION_MANAGER.requestQueryId();
- ExecutionResult result =
- COORDINATOR.executeForTreeModel(
- statement,
- queryId,
- SESSION_MANAGER.getSessionInfo(clientSession),
- "",
- partitionFetcher,
- schemaFetcher);
-
+ ExecutionResult result;
+ if (statement.isWriteToTable()) {
+ result =
+ COORDINATOR.executeForTableModel(
+ statement,
+ relationSqlParser,
+ clientSession,
+ queryId,
+ SESSION_MANAGER.getSessionInfo(clientSession),
+ "",
+ metadata,
+ config.getConnectionTimeoutInMS());
+ } else {
+ result =
+ COORDINATOR.executeForTreeModel(
+ statement,
+ queryId,
+ SESSION_MANAGER.getSessionInfo(clientSession),
+ "",
+ partitionFetcher,
+ schemaFetcher);
+ }
return result.status;
} catch (IoTDBException e) {
return onIoTDBException(e, OperationType.INSERT_TABLET, e.getErrorCode());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
index ce171b74713..03738bc0c49 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
@@ -81,6 +81,8 @@ public class MPPQueryContext {
// constructing some Expression and PlanNode.
private final MemoryReservationManager memoryReservationManager;
+ private boolean isTableQuery = false;
+
public MPPQueryContext(QueryId queryId) {
this.queryId = queryId;
this.endPointBlackList = new LinkedList<>();
@@ -329,4 +331,12 @@ public class MPPQueryContext {
}
// endregion
+
+ public boolean isTableQuery() {
+ return isTableQuery;
+ }
+
+ public void setTableQuery(boolean tableQuery) {
+ isTableQuery = tableQuery;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
index 41dc339c23d..a1ee9be5cae 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
@@ -62,6 +62,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager;
@@ -208,6 +209,12 @@ public class RegionWriteExecutor {
return executeDataInsert(node, context);
}
+ @Override
+ public RegionExecutionResult visitRelationalInsertTablet(
+ RelationalInsertTabletNode node, WritePlanNodeExecutionContext context) {
+ return executeDataInsert(node, context);
+ }
+
@Override
public RegionExecutionResult visitInsertRows(
InsertRowsNode node, WritePlanNodeExecutionContext context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java
index 2f55743854e..4b0ecf27c37 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java
@@ -68,6 +68,9 @@ public class CollectOperator implements ProcessOperator {
@Override
public ListenableFuture<?> isBlocked() {
+ if (currentIndex >= children.size()) {
+ return NOT_BLOCKED;
+ }
return children.get(currentIndex).isBlocked();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java
index 8dc1ce33ed8..b745115d06b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java
@@ -181,6 +181,9 @@ public class TableDeviceQuerySource implements ISchemaSource<IDeviceSchemaInfo>
}
private SchemaFilter getExecutableIdFuzzyFilter(Expression idFuzzyExpression) {
+ if (idFuzzyExpression == null) {
+ return null;
+ }
ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor();
return visitor.process(
idFuzzyExpression, new ConvertSchemaPredicateToFilterVisitor.Context(table));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
index a036d886e77..afe23cf53ae 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
@@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB;
@@ -60,6 +61,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement;
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
@@ -260,6 +262,54 @@ public class Coordinator {
startTime)));
}
+ public ExecutionResult executeForTableModel(
+ Statement statement,
+ SqlParser sqlParser,
+ IClientSession clientSession,
+ long queryId,
+ SessionInfo session,
+ String sql,
+ Metadata metadata,
+ long timeOut) {
+ return execution(
+ queryId,
+ session,
+ sql,
+ ((queryContext, startTime) ->
+ createQueryExecutionForTableModel(
+ statement,
+ sqlParser,
+ clientSession,
+ queryContext,
+ metadata,
+ timeOut > 0 ? timeOut : CONFIG.getQueryTimeoutThreshold(),
+ startTime)));
+ }
+
+ private IQueryExecution createQueryExecutionForTableModel(
+ Statement statement,
+ SqlParser sqlParser,
+ IClientSession clientSession,
+ MPPQueryContext queryContext,
+ Metadata metadata,
+ long timeOut,
+ long startTime) {
+ queryContext.setTableQuery(true);
+ queryContext.setTimeOut(timeOut);
+ queryContext.setStartTime(startTime);
+ RelationalModelPlanner relationalModelPlanner =
+ new RelationalModelPlanner(
+ statement.toRelationalStatement(queryContext),
+ sqlParser,
+ metadata,
+ executor,
+ writeOperationExecutor,
+ scheduledExecutor,
+ SYNC_INTERNAL_SERVICE_CLIENT_MANAGER,
+ ASYNC_INTERNAL_SERVICE_CLIENT_MANAGER);
+ return new QueryExecution(relationalModelPlanner, queryContext, executor);
+ }
+
private IQueryExecution createQueryExecutionForTableModel(
org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement statement,
SqlParser sqlParser,
@@ -268,6 +318,7 @@ public class Coordinator {
Metadata metadata,
long timeOut,
long startTime) {
+ queryContext.setTableQuery(true);
queryContext.setTimeOut(timeOut);
queryContext.setStartTime(startTime);
if (statement instanceof DropDB
@@ -281,13 +332,17 @@ public class Coordinator {
|| statement instanceof ShowCluster
|| statement instanceof ShowRegions
|| statement instanceof ShowDataNodes
- || statement instanceof ShowConfigNodes) {
+ || statement instanceof ShowConfigNodes
+ || statement instanceof Flush) {
return new ConfigExecution(
queryContext,
null,
executor,
statement.accept(new TableConfigTaskVisitor(clientSession, metadata), queryContext));
}
+ if (statement instanceof WrappedInsertStatement) {
+ ((WrappedInsertStatement) statement).setContext(queryContext);
+ }
RelationalModelPlanner relationalModelPlanner =
new RelationalModelPlanner(
statement,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
index ecf99c7b383..d6b70be0955 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
@@ -82,6 +82,8 @@ public class Analysis implements IAnalysis {
// Common Analysis
/////////////////////////////////////////////////////////////////////////////////////////////////
+ private String databaseName;
+
// Statement
private Statement statement;
@@ -374,11 +376,11 @@ public class Analysis implements IAnalysis {
return (QueryStatement) statement;
}
- public Statement getStatement() {
+ public Statement getTreeStatement() {
return statement;
}
- public void setStatement(Statement statement) {
+ public void setRealStatement(Statement statement) {
this.statement = statement;
}
@@ -482,7 +484,7 @@ public class Analysis implements IAnalysis {
public TsBlock constructResultForMemorySource(MPPQueryContext context) {
StatementMemorySource memorySource =
new StatementMemorySourceVisitor()
- .process(getStatement(), new StatementMemorySourceContext(context, this));
+ .process(getTreeStatement(), new StatementMemorySourceContext(context, this));
setRespDatasetHeader(memorySource.getDatasetHeader());
return memorySource.getTsBlock();
}
@@ -992,4 +994,14 @@ public class Analysis implements IAnalysis {
public boolean fromWhere(FilterNode filterNode) {
return fromWhereFilterNodes.contains(NodeRef.of(filterNode));
}
+
+ @Override
+ public void setDatabaseName(String database) {
+ this.databaseName = database;
+ }
+
+ @Override
+ public String getDatabaseName() {
+ return databaseName;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
new file mode 100644
index 00000000000..6191ec5377c
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze;
+
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.partition.DataPartition;
+import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
+import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class AnalyzeUtils {
+
+ private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS =
+ PerformanceOverviewMetrics.getInstance();
+ private static final Logger LOGGER = LoggerFactory.getLogger(AnalyzeUtils.class);
+
+ private AnalyzeUtils() {
+ // util class
+ }
+
+ public static InsertBaseStatement analyzeInsert(
+ MPPQueryContext context,
+ InsertBaseStatement insertBaseStatement,
+ Runnable schemaValidation,
+ DataPartitionQueryFunc partitionFetcher,
+ DataPartitionQueryParamComputation partitionQueryParamComputation,
+ IAnalysis analysis,
+ boolean removeLogicalView) {
+ context.setQueryType(QueryType.WRITE);
+ insertBaseStatement.semanticCheck();
+ validateSchema(analysis, insertBaseStatement, schemaValidation);
+
+ InsertBaseStatement realStatement =
+ removeLogicalView ? removeLogicalView(analysis, insertBaseStatement) : insertBaseStatement;
+ if (analysis.isFinishQueryAfterAnalyze()) {
+ return realStatement;
+ }
+ analysis.setRealStatement(realStatement);
+
+ analyzeDataPartition(
+ analysis,
+ partitionQueryParamComputation.compute(realStatement, context),
+ context.getSession().getUserName(),
+ partitionFetcher);
+ return realStatement;
+ }
+
+ public static List<DataPartitionQueryParam> computeTableDataPartitionParams(
+ InsertBaseStatement statement, MPPQueryContext context) {
+ if (statement instanceof InsertTabletStatement) {
+ InsertTabletStatement insertTabletStatement = (InsertTabletStatement) statement;
+ Map<IDeviceID, Set<TTimePartitionSlot>> timePartitionSlotMap = new HashMap<>();
+ for (int i = 0; i < insertTabletStatement.getRowCount(); i++) {
+ timePartitionSlotMap
+ .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new HashSet<>())
+ .add(insertTabletStatement.getTimePartitionSlot(i));
+ }
+ return computeDataPartitionParams(
+ timePartitionSlotMap, context.getSession().getDatabaseName().orElse(null));
+ } else if (statement instanceof InsertMultiTabletsStatement) {
+ InsertMultiTabletsStatement insertMultiTabletsStatement =
+ (InsertMultiTabletsStatement) statement;
+ Map<IDeviceID, Set<TTimePartitionSlot>> timePartitionSlotMap = new HashMap<>();
+ for (InsertTabletStatement insertTabletStatement :
+ insertMultiTabletsStatement.getInsertTabletStatementList()) {
+ for (int i = 0; i < insertTabletStatement.getRowCount(); i++) {
+ timePartitionSlotMap
+ .computeIfAbsent(insertTabletStatement.getTableDeviceID(i), id -> new HashSet<>())
+ .add(insertTabletStatement.getTimePartitionSlot(i));
+ }
+ }
+ return computeDataPartitionParams(
+ timePartitionSlotMap, context.getSession().getDatabaseName().orElse(null));
+ } else if (statement instanceof InsertRowStatement) {
+ InsertRowStatement insertRowStatement = (InsertRowStatement) statement;
+ return computeDataPartitionParams(
+ Collections.singletonMap(
+ insertRowStatement.getTableDeviceID(),
+ Collections.singleton(insertRowStatement.getTimePartitionSlot())),
+ context.getSession().getDatabaseName().orElse(null));
+ } else if (statement instanceof InsertRowsStatement) {
+ InsertRowsStatement insertRowsStatement = (InsertRowsStatement) statement;
+ Map<IDeviceID, Set<TTimePartitionSlot>> timePartitionSlotMap = new HashMap<>();
+ for (InsertRowStatement insertRowStatement :
+ insertRowsStatement.getInsertRowStatementList()) {
+ timePartitionSlotMap
+ .computeIfAbsent(insertRowStatement.getTableDeviceID(), id -> new HashSet<>())
+ .add(insertRowStatement.getTimePartitionSlot());
+ }
+ return computeDataPartitionParams(
+ timePartitionSlotMap, context.getSession().getDatabaseName().orElse(null));
+ }
+ throw new UnsupportedOperationException("computeDataPartitionParams for " + statement);
+ }
+
+ public static List<DataPartitionQueryParam> computeTreeDataPartitionParams(
+ InsertBaseStatement statement, MPPQueryContext context) {
+ if (statement instanceof InsertTabletStatement) {
+ DataPartitionQueryParam dataPartitionQueryParam =
+ getTreeDataPartitionQueryParam((InsertTabletStatement) statement, context);
+ return Collections.singletonList(dataPartitionQueryParam);
+ } else if (statement instanceof InsertMultiTabletsStatement) {
+ InsertMultiTabletsStatement insertMultiTabletsStatement =
+ (InsertMultiTabletsStatement) statement;
+ Map<IDeviceID, Set<TTimePartitionSlot>> dataPartitionQueryParamMap = new HashMap<>();
+ for (InsertTabletStatement insertTabletStatement :
+ insertMultiTabletsStatement.getInsertTabletStatementList()) {
+ Set<TTimePartitionSlot> timePartitionSlotSet =
+ dataPartitionQueryParamMap.computeIfAbsent(
+ insertTabletStatement.getDevicePath().getIDeviceIDAsFullDevice(),
+ k -> new HashSet<>());
+ timePartitionSlotSet.addAll(insertTabletStatement.getTimePartitionSlots());
+ }
+ return computeDataPartitionParams(
+ dataPartitionQueryParamMap, context.getSession().getDatabaseName().orElse(null));
+ } else if (statement instanceof InsertRowsStatement) {
+ final InsertRowsStatement insertRowsStatement = (InsertRowsStatement) statement;
+ Map<IDeviceID, Set<TTimePartitionSlot>> dataPartitionQueryParamMap = new HashMap<>();
+ for (InsertRowStatement insertRowStatement :
+ insertRowsStatement.getInsertRowStatementList()) {
+ Set<TTimePartitionSlot> timePartitionSlotSet =
+ dataPartitionQueryParamMap.computeIfAbsent(
+ insertRowStatement.getDevicePath().getIDeviceIDAsFullDevice(),
+ k -> new HashSet<>());
+ timePartitionSlotSet.add(insertRowStatement.getTimePartitionSlot());
+ }
+ return computeDataPartitionParams(
+ dataPartitionQueryParamMap, context.getSession().getDatabaseName().orElse(null));
+ }
+ throw new UnsupportedOperationException("computeDataPartitionParams for " + statement);
+ }
+
+ private static DataPartitionQueryParam getTreeDataPartitionQueryParam(
+ InsertTabletStatement statement, MPPQueryContext context) {
+ DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam();
+ dataPartitionQueryParam.setDeviceID(statement.getDevicePath().getIDeviceIDAsFullDevice());
+ dataPartitionQueryParam.setTimePartitionSlotList(statement.getTimePartitionSlots());
+ dataPartitionQueryParam.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
+ return dataPartitionQueryParam;
+ }
+
+ public static List<DataPartitionQueryParam> computeDataPartitionParams(
+ Map<IDeviceID, Set<TTimePartitionSlot>> dataPartitionQueryParamMap, String databaseName) {
+ List<DataPartitionQueryParam> dataPartitionQueryParams = new ArrayList<>();
+ for (Map.Entry<IDeviceID, Set<TTimePartitionSlot>> entry :
+ dataPartitionQueryParamMap.entrySet()) {
+ DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam();
+ dataPartitionQueryParam.setDeviceID(entry.getKey());
+ dataPartitionQueryParam.setTimePartitionSlotList(new ArrayList<>(entry.getValue()));
+ dataPartitionQueryParam.setDatabaseName(databaseName);
+ dataPartitionQueryParams.add(dataPartitionQueryParam);
+ }
+ return dataPartitionQueryParams;
+ }
+
+ public static void validateSchema(
+ IAnalysis analysis, InsertBaseStatement insertStatement, Runnable schemaValidation) {
+ final long startTime = System.nanoTime();
+ try {
+ schemaValidation.run();
+ } catch (SemanticException e) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ if (e.getCause() instanceof IoTDBException) {
+ IoTDBException exception = (IoTDBException) e.getCause();
+ analysis.setFailStatus(
+ RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()));
+ } else {
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage()));
+ }
+ } finally {
+ PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost(System.nanoTime() - startTime);
+ }
+ boolean hasFailedMeasurement = insertStatement.hasFailedMeasurements();
+ String partialInsertMessage;
+ if (hasFailedMeasurement) {
+ partialInsertMessage =
+ String.format(
+ "Fail to insert measurements %s caused by %s",
+ insertStatement.getFailedMeasurements(), insertStatement.getFailedMessages());
+ LOGGER.warn(partialInsertMessage);
+ analysis.setFailStatus(
+ RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(), partialInsertMessage));
+ }
+ }
+
+ public static InsertBaseStatement removeLogicalView(
+ IAnalysis analysis, InsertBaseStatement insertBaseStatement) {
+ try {
+ return insertBaseStatement.removeLogicalView();
+ } catch (SemanticException e) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ if (e.getCause() instanceof IoTDBException) {
+ IoTDBException exception = (IoTDBException) e.getCause();
+ analysis.setFailStatus(
+ RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()));
+ } else {
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage()));
+ }
+ return insertBaseStatement;
+ }
+ }
+
+ /** get analysis according to statement and params */
+ public static void analyzeDataPartition(
+ IAnalysis analysis,
+ List<DataPartitionQueryParam> dataPartitionQueryParams,
+ String userName,
+ DataPartitionQueryFunc partitionQueryFunc) {
+
+ DataPartition dataPartition =
+ partitionQueryFunc.queryDataPartition(dataPartitionQueryParams, userName);
+ if (dataPartition.isEmpty()) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(
+ RpcUtils.getStatus(
+ TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(),
+ "Database not exists and failed to create automatically "
+ + "because enable_auto_create_schema is FALSE."));
+ }
+ analysis.setDataPartitionInfo(dataPartition);
+ }
+
+ public interface DataPartitionQueryFunc {
+
+ DataPartition queryDataPartition(
+ List<DataPartitionQueryParam> dataPartitionQueryParams, String userName);
+ }
+
+ public interface DataPartitionQueryParamComputation {
+
+ List<DataPartitionQueryParam> compute(InsertBaseStatement statement, MPPQueryContext context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 2b6c152500b..858898386b2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.partition.DataPartition;
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
@@ -191,6 +190,8 @@ import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant
import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER;
import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER;
import static org.apache.iotdb.db.queryengine.metric.load.LoadTsFileCostMetricsSet.ANALYSIS;
+import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils.removeLogicalView;
+import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils.validateSchema;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.bindSchemaForExpression;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.getMeasurementExpression;
@@ -243,8 +244,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
@Override
public Analysis visitExplain(ExplainStatement explainStatement, MPPQueryContext context) {
Analysis analysis = visitQuery(explainStatement.getQueryStatement(), context);
- analysis.setStatement(explainStatement);
+ analysis.setRealStatement(explainStatement);
analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
return analysis;
}
@@ -253,12 +255,13 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
ExplainAnalyzeStatement explainAnalyzeStatement, MPPQueryContext context) {
Analysis analysis = visitQuery(explainAnalyzeStatement.getQueryStatement(), context);
context.setExplainAnalyze(true);
- analysis.setStatement(explainAnalyzeStatement);
+ analysis.setRealStatement(explainAnalyzeStatement);
analysis.setRespDatasetHeader(
new DatasetHeader(
Collections.singletonList(
new ColumnHeader(ColumnHeaderConstant.EXPLAIN_ANALYZE, TSDataType.TEXT, null)),
true));
+ analysis.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
return analysis;
}
@@ -408,7 +411,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
(QueryStatement)
concatPathRewriter.rewrite(
queryStatement, new PathPatternTree(queryStatement.useWildcard()), context);
- analysis.setStatement(queryStatement);
+ analysis.setRealStatement(queryStatement);
// request schema fetch API
long startTime = System.nanoTime();
@@ -580,8 +583,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
analysis.setUseLogicalView(useLogicalView);
if (useLogicalView
- && analysis.getStatement() instanceof QueryStatement
- && (((QueryStatement) analysis.getStatement()).isGroupByTag())) {
+ && analysis.getTreeStatement() instanceof QueryStatement
+ && (((QueryStatement) analysis.getTreeStatement()).isGroupByTag())) {
throw new SemanticException("Views cannot be used in GROUP BY TAGS query yet.");
}
@@ -2394,7 +2397,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
Analysis analysis = new Analysis();
- analysis.setStatement(createTimeSeriesStatement);
+ analysis.setRealStatement(createTimeSeriesStatement);
checkIsTableCompatible(createTimeSeriesStatement.getPath(), context);
checkIsTemplateCompatible(
@@ -2506,7 +2509,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
Analysis analysis = new Analysis();
- analysis.setStatement(createAlignedTimeSeriesStatement);
+ analysis.setRealStatement(createAlignedTimeSeriesStatement);
checkIsTableCompatible(createAlignedTimeSeriesStatement.getDevicePath(), context);
checkIsTemplateCompatible(
@@ -2536,7 +2539,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(internalCreateTimeSeriesStatement);
+ analysis.setRealStatement(internalCreateTimeSeriesStatement);
checkIsTableCompatible(internalCreateTimeSeriesStatement.getDevicePath(), context);
checkIsTemplateCompatible(
internalCreateTimeSeriesStatement.getDevicePath(),
@@ -2566,7 +2569,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(internalCreateMultiTimeSeriesStatement);
+ analysis.setRealStatement(internalCreateMultiTimeSeriesStatement);
PathPatternTree pathPatternTree = new PathPatternTree();
DataNodeSchemaLockManager.getInstance().takeReadLock(SchemaLockType.TIMESERIES_VS_TEMPLATE);
@@ -2592,7 +2595,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
CreateMultiTimeSeriesStatement createMultiTimeSeriesStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(createMultiTimeSeriesStatement);
+ analysis.setRealStatement(createMultiTimeSeriesStatement);
analyzeSchemaProps(createMultiTimeSeriesStatement.getPropsList());
@@ -2623,7 +2626,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
AlterTimeSeriesStatement alterTimeSeriesStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(alterTimeSeriesStatement);
+ analysis.setRealStatement(alterTimeSeriesStatement);
Pair<Template, PartialPath> templateInfo =
schemaFetcher.checkTemplateSetAndPreSetInfo(
@@ -2649,34 +2652,16 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
@Override
public Analysis visitInsertTablet(
InsertTabletStatement insertTabletStatement, MPPQueryContext context) {
- context.setQueryType(QueryType.WRITE);
- insertTabletStatement.semanticCheck();
Analysis analysis = new Analysis();
- validateSchema(analysis, insertTabletStatement, context);
- InsertBaseStatement realStatement = removeLogicalView(analysis, insertTabletStatement);
- if (analysis.isFinishQueryAfterAnalyze()) {
- return analysis;
- }
- analysis.setStatement(realStatement);
-
- if (realStatement instanceof InsertTabletStatement) {
- InsertTabletStatement realInsertTabletStatement = (InsertTabletStatement) realStatement;
- DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam();
- dataPartitionQueryParam.setDeviceID(
- realInsertTabletStatement.getDevicePath().getIDeviceIDAsFullDevice());
- dataPartitionQueryParam.setTimePartitionSlotList(
- realInsertTabletStatement.getTimePartitionSlots());
-
- return getAnalysisForWriting(
- analysis,
- Collections.singletonList(dataPartitionQueryParam),
- context.getSession().getUserName());
- } else {
- return computeAnalysisForMultiTablets(
- analysis,
- (InsertMultiTabletsStatement) realStatement,
- context.getSession().getUserName());
- }
+ AnalyzeUtils.analyzeInsert(
+ context,
+ insertTabletStatement,
+ () -> SchemaValidator.validate(schemaFetcher, insertTabletStatement, context),
+ partitionFetcher::getOrCreateDataPartition,
+ AnalyzeUtils::computeTreeDataPartitionParams,
+ analysis,
+ true);
+ return analysis;
}
@Override
@@ -2684,12 +2669,15 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
insertRowStatement.semanticCheck();
Analysis analysis = new Analysis();
- validateSchema(analysis, insertRowStatement, context);
+ validateSchema(
+ analysis,
+ insertRowStatement,
+ () -> SchemaValidator.validate(schemaFetcher, insertRowStatement, context));
InsertBaseStatement realInsertStatement = removeLogicalView(analysis, insertRowStatement);
if (analysis.isFinishQueryAfterAnalyze()) {
return analysis;
}
- analysis.setStatement(realInsertStatement);
+ analysis.setRealStatement(realInsertStatement);
if (realInsertStatement instanceof InsertRowStatement) {
InsertRowStatement realInsertRowStatement = (InsertRowStatement) realInsertStatement;
@@ -2699,36 +2687,19 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
dataPartitionQueryParam.setTimePartitionSlotList(
Collections.singletonList(realInsertRowStatement.getTimePartitionSlot()));
- return getAnalysisForWriting(
+ AnalyzeUtils.analyzeDataPartition(
analysis,
Collections.singletonList(dataPartitionQueryParam),
- context.getSession().getUserName());
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
} else {
- return computeAnalysisForInsertRows(
- analysis, (InsertRowsStatement) realInsertStatement, context.getSession().getUserName());
- }
- }
-
- private Analysis computeAnalysisForInsertRows(
- Analysis analysis, InsertRowsStatement insertRowsStatement, String userName) {
- Map<IDeviceID, Set<TTimePartitionSlot>> dataPartitionQueryParamMap = new HashMap<>();
- for (InsertRowStatement insertRowStatement : insertRowsStatement.getInsertRowStatementList()) {
- Set<TTimePartitionSlot> timePartitionSlotSet =
- dataPartitionQueryParamMap.computeIfAbsent(
- insertRowStatement.getDevicePath().getIDeviceIDAsFullDevice(), k -> new HashSet<>());
- timePartitionSlotSet.add(insertRowStatement.getTimePartitionSlot());
- }
-
- List<DataPartitionQueryParam> dataPartitionQueryParams = new ArrayList<>();
- for (Map.Entry<IDeviceID, Set<TTimePartitionSlot>> entry :
- dataPartitionQueryParamMap.entrySet()) {
- DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam();
- dataPartitionQueryParam.setDeviceID(entry.getKey());
- dataPartitionQueryParam.setTimePartitionSlotList(new ArrayList<>(entry.getValue()));
- dataPartitionQueryParams.add(dataPartitionQueryParam);
+ AnalyzeUtils.analyzeDataPartition(
+ analysis,
+ AnalyzeUtils.computeTreeDataPartitionParams(realInsertStatement, context),
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
}
-
- return getAnalysisForWriting(analysis, dataPartitionQueryParams, userName);
+ return analysis;
}
@Override
@@ -2737,40 +2708,23 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
insertRowsStatement.semanticCheck();
Analysis analysis = new Analysis();
- validateSchema(analysis, insertRowsStatement, context);
+ validateSchema(
+ analysis,
+ insertRowsStatement,
+ () -> SchemaValidator.validate(schemaFetcher, insertRowsStatement, context));
InsertRowsStatement realInsertRowsStatement =
(InsertRowsStatement) removeLogicalView(analysis, insertRowsStatement);
if (analysis.isFinishQueryAfterAnalyze()) {
return analysis;
}
- analysis.setStatement(realInsertRowsStatement);
+ analysis.setRealStatement(realInsertRowsStatement);
- return computeAnalysisForInsertRows(
- analysis, realInsertRowsStatement, context.getSession().getUserName());
- }
-
- private Analysis computeAnalysisForMultiTablets(
- Analysis analysis, InsertMultiTabletsStatement insertMultiTabletsStatement, String userName) {
- Map<IDeviceID, Set<TTimePartitionSlot>> dataPartitionQueryParamMap = new HashMap<>();
- for (InsertTabletStatement insertTabletStatement :
- insertMultiTabletsStatement.getInsertTabletStatementList()) {
- Set<TTimePartitionSlot> timePartitionSlotSet =
- dataPartitionQueryParamMap.computeIfAbsent(
- insertTabletStatement.getDevicePath().getIDeviceIDAsFullDevice(),
- k -> new HashSet<>());
- timePartitionSlotSet.addAll(insertTabletStatement.getTimePartitionSlots());
- }
-
- List<DataPartitionQueryParam> dataPartitionQueryParams = new ArrayList<>();
- for (Map.Entry<IDeviceID, Set<TTimePartitionSlot>> entry :
- dataPartitionQueryParamMap.entrySet()) {
- DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam();
- dataPartitionQueryParam.setDeviceID(entry.getKey());
- dataPartitionQueryParam.setTimePartitionSlotList(new ArrayList<>(entry.getValue()));
- dataPartitionQueryParams.add(dataPartitionQueryParam);
- }
-
- return getAnalysisForWriting(analysis, dataPartitionQueryParams, userName);
+ AnalyzeUtils.analyzeDataPartition(
+ analysis,
+ AnalyzeUtils.computeTreeDataPartitionParams(realInsertRowsStatement, context),
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
+ return analysis;
}
@Override
@@ -2779,16 +2733,23 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
insertMultiTabletsStatement.semanticCheck();
Analysis analysis = new Analysis();
- validateSchema(analysis, insertMultiTabletsStatement, context);
+ validateSchema(
+ analysis,
+ insertMultiTabletsStatement,
+ () -> SchemaValidator.validate(schemaFetcher, insertMultiTabletsStatement, context));
InsertMultiTabletsStatement realStatement =
(InsertMultiTabletsStatement) removeLogicalView(analysis, insertMultiTabletsStatement);
if (analysis.isFinishQueryAfterAnalyze()) {
return analysis;
}
- analysis.setStatement(realStatement);
+ analysis.setRealStatement(realStatement);
- return computeAnalysisForMultiTablets(
- analysis, realStatement, context.getSession().getUserName());
+ AnalyzeUtils.analyzeDataPartition(
+ analysis,
+ AnalyzeUtils.computeTreeDataPartitionParams(realStatement, context),
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
+ return analysis;
}
@Override
@@ -2797,13 +2758,16 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
insertRowsOfOneDeviceStatement.semanticCheck();
Analysis analysis = new Analysis();
- validateSchema(analysis, insertRowsOfOneDeviceStatement, context);
+ validateSchema(
+ analysis,
+ insertRowsOfOneDeviceStatement,
+ () -> SchemaValidator.validate(schemaFetcher, insertRowsOfOneDeviceStatement, context));
InsertBaseStatement realInsertStatement =
removeLogicalView(analysis, insertRowsOfOneDeviceStatement);
if (analysis.isFinishQueryAfterAnalyze()) {
return analysis;
}
- analysis.setStatement(realInsertStatement);
+ analysis.setRealStatement(realInsertStatement);
if (realInsertStatement instanceof InsertRowsOfOneDeviceStatement) {
InsertRowsOfOneDeviceStatement realStatement =
@@ -2812,74 +2776,33 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
dataPartitionQueryParam.setDeviceID(realStatement.getDevicePath().getIDeviceIDAsFullDevice());
dataPartitionQueryParam.setTimePartitionSlotList(realStatement.getTimePartitionSlots());
- return getAnalysisForWriting(
+ AnalyzeUtils.analyzeDataPartition(
analysis,
Collections.singletonList(dataPartitionQueryParam),
- context.getSession().getUserName());
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
} else {
- return computeAnalysisForInsertRows(
- analysis, (InsertRowsStatement) realInsertStatement, context.getSession().getUserName());
+ AnalyzeUtils.analyzeDataPartition(
+ analysis,
+ AnalyzeUtils.computeTreeDataPartitionParams(realInsertStatement, context),
+ context.getSession().getUserName(),
+ partitionFetcher::getOrCreateDataPartition);
}
+ return analysis;
}
@Override
public Analysis visitPipeEnrichedStatement(
PipeEnrichedStatement pipeEnrichedStatement, MPPQueryContext context) {
Analysis analysis = pipeEnrichedStatement.getInnerStatement().accept(this, context);
+ analysis.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
// statement may be changed because of logical view
- pipeEnrichedStatement.setInnerStatement(analysis.getStatement());
- analysis.setStatement(pipeEnrichedStatement);
+ pipeEnrichedStatement.setInnerStatement(analysis.getTreeStatement());
+ analysis.setRealStatement(pipeEnrichedStatement);
return analysis;
}
- private void validateSchema(
- Analysis analysis, InsertBaseStatement insertStatement, MPPQueryContext context) {
- final long startTime = System.nanoTime();
- try {
- SchemaValidator.validate(schemaFetcher, insertStatement, context);
- } catch (SemanticException e) {
- analysis.setFinishQueryAfterAnalyze(true);
- if (e.getCause() instanceof IoTDBException) {
- IoTDBException exception = (IoTDBException) e.getCause();
- analysis.setFailStatus(
- RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()));
- } else {
- analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage()));
- }
- } finally {
- PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost(System.nanoTime() - startTime);
- }
- boolean hasFailedMeasurement = insertStatement.hasFailedMeasurements();
- String partialInsertMessage;
- if (hasFailedMeasurement) {
- partialInsertMessage =
- String.format(
- "Fail to insert measurements %s caused by %s",
- insertStatement.getFailedMeasurements(), insertStatement.getFailedMessages());
- logger.warn(partialInsertMessage);
- analysis.setFailStatus(
- RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(), partialInsertMessage));
- }
- }
-
- private InsertBaseStatement removeLogicalView(
- Analysis analysis, InsertBaseStatement insertBaseStatement) {
- try {
- return insertBaseStatement.removeLogicalView();
- } catch (SemanticException e) {
- analysis.setFinishQueryAfterAnalyze(true);
- if (e.getCause() instanceof IoTDBException) {
- IoTDBException exception = (IoTDBException) e.getCause();
- analysis.setFailStatus(
- RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()));
- } else {
- analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage()));
- }
- return insertBaseStatement;
- }
- }
-
@Override
public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
@@ -2905,24 +2828,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
}
- /** get analysis according to statement and params */
- private Analysis getAnalysisForWriting(
- Analysis analysis, List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) {
-
- DataPartition dataPartition =
- partitionFetcher.getOrCreateDataPartition(dataPartitionQueryParams, userName);
- if (dataPartition.isEmpty()) {
- analysis.setFinishQueryAfterAnalyze(true);
- analysis.setFailStatus(
- RpcUtils.getStatus(
- TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(),
- "Database not exists and failed to create automatically "
- + "because enable_auto_create_schema is FALSE."));
- }
- analysis.setDataPartitionInfo(dataPartition);
- return analysis;
- }
-
private boolean analyzeTimeseriesRegionScan(
WhereCondition timeCondition,
PathPatternTree patternTree,
@@ -2995,7 +2900,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowTimeSeries(
ShowTimeSeriesStatement showTimeSeriesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showTimeSeriesStatement);
+ analysis.setRealStatement(showTimeSeriesStatement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(showTimeSeriesStatement.getPathPattern());
@@ -3052,7 +2957,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowStorageGroup(
ShowDatabaseStatement showDatabaseStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showDatabaseStatement);
+ analysis.setRealStatement(showDatabaseStatement);
analysis.setRespDatasetHeader(
DatasetHeaderFactory.getShowStorageGroupHeader(showDatabaseStatement.isDetailed()));
return analysis;
@@ -3061,7 +2966,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
@Override
public Analysis visitShowTTL(ShowTTLStatement showTTLStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showTTLStatement);
+ analysis.setRealStatement(showTTLStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTTLHeader());
return analysis;
}
@@ -3129,7 +3034,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowDevices(
ShowDevicesStatement showDevicesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showDevicesStatement);
+ analysis.setRealStatement(showDevicesStatement);
if (showDevicesStatement.hasTimeCondition()) {
analyzeDeviceRegionScan(
@@ -3156,7 +3061,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowCluster(
ShowClusterStatement showClusterStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showClusterStatement);
+ analysis.setRealStatement(showClusterStatement);
if (showClusterStatement.isDetails()) {
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowClusterDetailsHeader());
} else {
@@ -3169,7 +3074,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitCountStorageGroup(
CountDatabaseStatement countDatabaseStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(countDatabaseStatement);
+ analysis.setRealStatement(countDatabaseStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getCountStorageGroupHeader());
return analysis;
}
@@ -3178,7 +3083,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitSeriesSchemaFetch(
SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(seriesSchemaFetchStatement);
+ analysis.setRealStatement(seriesSchemaFetchStatement);
SchemaPartition schemaPartition =
partitionFetcher.getSchemaPartition(seriesSchemaFetchStatement.getPatternTree());
@@ -3195,7 +3100,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitDeviceSchemaFetch(
DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(deviceSchemaFetchStatement);
+ analysis.setRealStatement(deviceSchemaFetchStatement);
PathPatternTree patternTree = new PathPatternTree();
for (PartialPath path : deviceSchemaFetchStatement.getPaths()) {
@@ -3216,7 +3121,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitCountDevices(
CountDevicesStatement countDevicesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(countDevicesStatement);
+ analysis.setRealStatement(countDevicesStatement);
if (countDevicesStatement.hasTimeCondition()) {
analyzeDeviceRegionScan(
@@ -3241,7 +3146,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitCountTimeSeries(
CountTimeSeriesStatement countTimeSeriesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(countTimeSeriesStatement);
+ analysis.setRealStatement(countTimeSeriesStatement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(countTimeSeriesStatement.getPathPattern());
@@ -3277,7 +3182,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitCountLevelTimeSeries(
CountLevelTimeSeriesStatement countLevelTimeSeriesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(countLevelTimeSeriesStatement);
+ analysis.setRealStatement(countLevelTimeSeriesStatement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(countLevelTimeSeriesStatement.getPathPattern());
@@ -3294,7 +3199,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
@Override
public Analysis visitCountNodes(CountNodesStatement countStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(countStatement);
+ analysis.setRealStatement(countStatement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(countStatement.getPathPattern());
@@ -3339,7 +3244,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowVersion(
ShowVersionStatement showVersionStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showVersionStatement);
+ analysis.setRealStatement(showVersionStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowVersionHeader());
analysis.setFinishQueryAfterAnalyze(true);
return analysis;
@@ -3348,7 +3253,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
private Analysis visitSchemaNodeManagementPartition(
Statement statement, PartialPath path, PathPatternTree scope, DatasetHeader header) {
Analysis analysis = new Analysis();
- analysis.setStatement(statement);
+ analysis.setRealStatement(statement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(path);
@@ -3373,7 +3278,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
DeleteDataStatement deleteDataStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(deleteDataStatement);
+ analysis.setRealStatement(deleteDataStatement);
PathPatternTree patternTree = new PathPatternTree();
deleteDataStatement.getPathList().forEach(patternTree::appendPathPattern);
@@ -3446,7 +3351,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
"Measurement under template is not allowed to have the same measurement name");
}
Analysis analysis = new Analysis();
- analysis.setStatement(createTemplateStatement);
+ analysis.setRealStatement(createTemplateStatement);
return analysis;
}
@@ -3455,7 +3360,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
ShowNodesInSchemaTemplateStatement showNodesInSchemaTemplateStatement,
MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showNodesInSchemaTemplateStatement);
+ analysis.setRealStatement(showNodesInSchemaTemplateStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowNodesInSchemaTemplateHeader());
return analysis;
}
@@ -3464,7 +3369,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowSchemaTemplate(
ShowSchemaTemplateStatement showSchemaTemplateStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showSchemaTemplateStatement);
+ analysis.setRealStatement(showSchemaTemplateStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowSchemaTemplateHeader());
return analysis;
}
@@ -3474,7 +3379,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
SetSchemaTemplateStatement setSchemaTemplateStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(setSchemaTemplateStatement);
+ analysis.setRealStatement(setSchemaTemplateStatement);
return analysis;
}
@@ -3482,7 +3387,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowPathSetTemplate(
ShowPathSetTemplateStatement showPathSetTemplateStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showPathSetTemplateStatement);
+ analysis.setRealStatement(showPathSetTemplateStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowPathSetTemplateHeader());
return analysis;
}
@@ -3492,7 +3397,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
ActivateTemplateStatement activateTemplateStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(activateTemplateStatement);
+ analysis.setRealStatement(activateTemplateStatement);
PartialPath activatePath = activateTemplateStatement.getPath();
@@ -3522,7 +3427,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
BatchActivateTemplateStatement batchActivateTemplateStatement, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(batchActivateTemplateStatement);
+ analysis.setRealStatement(batchActivateTemplateStatement);
Map<PartialPath, Pair<Template, PartialPath>> deviceTemplateSetInfoMap =
new HashMap<>(batchActivateTemplateStatement.getDevicePathList().size());
@@ -3558,7 +3463,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
Analysis analysis = new Analysis();
- analysis.setStatement(internalBatchActivateTemplateStatement);
+ analysis.setRealStatement(internalBatchActivateTemplateStatement);
PathPatternTree patternTree = new PathPatternTree();
for (PartialPath activatePath :
@@ -3579,7 +3484,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowPathsUsingTemplate(
ShowPathsUsingTemplateStatement showPathsUsingTemplateStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showPathsUsingTemplateStatement);
+ analysis.setRealStatement(showPathsUsingTemplateStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowPathsUsingTemplateHeader());
Pair<Template, List<PartialPath>> templateSetInfo =
@@ -3626,7 +3531,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowQueries(
ShowQueriesStatement showQueriesStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showQueriesStatement);
+ analysis.setRealStatement(showQueriesStatement);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowQueriesHeader());
analysis.setVirtualSource(true);
@@ -3698,7 +3603,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
CreateLogicalViewStatement createLogicalViewStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
context.setQueryType(QueryType.WRITE);
- analysis.setStatement(createLogicalViewStatement);
+ analysis.setRealStatement(createLogicalViewStatement);
if (createLogicalViewStatement.getViewExpressions() == null) {
// Analyze query in statement
@@ -3960,7 +3865,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
ShowLogicalViewStatement showLogicalViewStatement, MPPQueryContext context) {
context.setQueryType(QueryType.READ);
Analysis analysis = new Analysis();
- analysis.setStatement(showLogicalViewStatement);
+ analysis.setRealStatement(showLogicalViewStatement);
PathPatternTree patternTree = new PathPatternTree();
patternTree.appendPathPattern(showLogicalViewStatement.getPathPattern());
@@ -3977,7 +3882,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public Analysis visitShowCurrentTimestamp(
ShowCurrentTimestampStatement showCurrentTimestampStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
- analysis.setStatement(showCurrentTimestampStatement);
+ analysis.setRealStatement(showCurrentTimestampStatement);
analysis.setFinishQueryAfterAnalyze(true);
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowCurrentTimestampHeader());
return analysis;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analyzer.java
index f7ca04f7d05..7ae16ef25f7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analyzer.java
@@ -45,6 +45,10 @@ public class Analyzer {
long startTime = System.nanoTime();
Analysis analysis =
new AnalyzeVisitor(partitionFetcher, schemaFetcher).process(statement, context);
+ if (context.getSession() != null) {
+ // for test compatibility
+ analysis.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
+ }
if (statement.isQuery()) {
QueryPlanCostMetricSet.getInstance().recordPlanCost(ANALYZER, System.nanoTime() - startTime);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
index 1f4f63f6d2d..ad5e19bd7c7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.ConfigRegionId;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.IoTDBException;
@@ -35,6 +36,7 @@ import org.apache.iotdb.commons.partition.SchemaPartition;
import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TSchemaNodeManagementReq;
@@ -44,6 +46,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.exception.sql.StatementAnalyzeException;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
@@ -64,8 +67,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
-import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
+import java.util.stream.Collectors;
public class ClusterPartitionFetcher implements IPartitionFetcher {
@@ -98,23 +100,44 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
@Override
public SchemaPartition getSchemaPartition(PathPatternTree patternTree) {
- return getOrCreateSchemaPartition(patternTree, false, null);
+ try (ConfigNodeClient client =
+ configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
+ patternTree.constructTree();
+ List<IDeviceID> deviceIDs = patternTree.getAllDevicePatterns();
+ Map<String, List<IDeviceID>> storageGroupToDeviceMap =
+ partitionCache.getDatabaseToDevice(deviceIDs, true, false, null);
+ SchemaPartition schemaPartition = partitionCache.getSchemaPartition(storageGroupToDeviceMap);
+ if (null == schemaPartition) {
+ TSchemaPartitionTableResp schemaPartitionTableResp =
+ client.getSchemaPartitionTable(constructSchemaPartitionReq(patternTree));
+ if (schemaPartitionTableResp.getStatus().getCode()
+ == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ schemaPartition = parseSchemaPartitionTableResp(schemaPartitionTableResp);
+ partitionCache.updateSchemaPartitionCache(
+ schemaPartitionTableResp.getSchemaPartitionTable());
+ } else {
+ throw new RuntimeException(
+ new IoTDBException(
+ schemaPartitionTableResp.getStatus().getMessage(),
+ schemaPartitionTableResp.getStatus().getCode()));
+ }
+ }
+ return schemaPartition;
+ } catch (ClientManagerException | TException e) {
+ throw new StatementAnalyzeException(
+ "An error occurred when executing getSchemaPartition():" + e.getMessage());
+ }
}
@Override
public SchemaPartition getOrCreateSchemaPartition(PathPatternTree patternTree, String userName) {
- return getOrCreateSchemaPartition(patternTree, true, userName);
- }
-
- private SchemaPartition getOrCreateSchemaPartition(
- PathPatternTree patternTree, boolean isAutoCreate, String userName) {
try (ConfigNodeClient client =
configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
patternTree.constructTree();
List<IDeviceID> deviceIDs = patternTree.getAllDevicePatterns();
- Map<String, List<IDeviceID>> databaseToDevice =
- partitionCache.getDatabaseToDevice(deviceIDs, true, isAutoCreate, userName);
- SchemaPartition schemaPartition = partitionCache.getSchemaPartition(databaseToDevice);
+ Map<String, List<IDeviceID>> storageGroupToDeviceMap =
+ partitionCache.getDatabaseToDevice(deviceIDs, true, true, userName);
+ SchemaPartition schemaPartition = partitionCache.getSchemaPartition(storageGroupToDeviceMap);
if (null == schemaPartition) {
TSchemaPartitionTableResp schemaPartitionTableResp =
client.getOrCreateSchemaPartitionTable(constructSchemaPartitionReq(patternTree));
@@ -293,10 +316,17 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
SchemaPartition schemaPartition =
partitionCache.getSchemaPartition(Collections.singletonMap(database, deviceIDs));
if (null == schemaPartition) {
- PathPatternTree tree = new PathPatternTree();
- tree.appendPathPattern(new PartialPath(database + "." + MULTI_LEVEL_PATH_WILDCARD));
+ List<TSeriesPartitionSlot> partitionSlots =
+ deviceIDs.stream()
+ .map(partitionExecutor::getSeriesPartitionSlot)
+ .distinct()
+ .collect(Collectors.toList());
TSchemaPartitionTableResp schemaPartitionTableResp =
- client.getSchemaPartitionTable(constructSchemaPartitionReq(tree));
+ isAutoCreate
+ ? client.getOrCreateSchemaPartitionTableWithSlots(
+ Collections.singletonMap(database, partitionSlots))
+ : client.getSchemaPartitionTableWithSlots(
+ Collections.singletonMap(database, partitionSlots));
if (schemaPartitionTableResp.getStatus().getCode()
== TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
schemaPartition = parseSchemaPartitionTableResp(schemaPartitionTableResp);
@@ -310,7 +340,7 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
}
}
return schemaPartition;
- } catch (ClientManagerException | TException | IllegalPathException e) {
+ } catch (ClientManagerException | TException e) {
throw new StatementAnalyzeException(
"An error occurred when executing getSchemaPartition():" + e.getMessage());
}
@@ -318,7 +348,17 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
@Override
public SchemaPartition getSchemaPartition(String database) {
- return partitionCache.getSchemaPartition(database);
+ PathPatternTree patternTree = new PathPatternTree();
+ try {
+ patternTree.appendPathPattern(
+ new PartialPath(
+ PathUtils.qualifyDatabaseName(database)
+ + IoTDBConstant.PATH_SEPARATOR
+ + IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD));
+ } catch (IllegalPathException e) {
+ throw new SemanticException(e);
+ }
+ return getSchemaPartition(patternTree);
}
/** split data partition query param by database */
@@ -330,13 +370,23 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) {
deviceIDs.add(dataPartitionQueryParam.getDeviceID());
}
- Map<IDeviceID, String> deviceToDatabase =
- partitionCache.getDeviceToDatabase(deviceIDs, true, isAutoCreate, userName);
+ Map<IDeviceID, String> deviceToDatabase = null;
Map<String, List<DataPartitionQueryParam>> result = new HashMap<>();
for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) {
IDeviceID deviceID = dataPartitionQueryParam.getDeviceID();
- if (deviceToDatabase.containsKey(deviceID)) {
- String database = deviceToDatabase.get(deviceID);
+ String database = null;
+ if (dataPartitionQueryParam.getDatabaseName() == null) {
+ if (deviceToDatabase == null) {
+ deviceToDatabase =
+ partitionCache.getDeviceToDatabase(deviceIDs, true, isAutoCreate, userName);
+ }
+ if (deviceToDatabase.containsKey(deviceID)) {
+ database = deviceToDatabase.get(deviceID);
+ }
+ } else {
+ database = dataPartitionQueryParam.getDatabaseName();
+ }
+ if (database != null) {
result.computeIfAbsent(database, key -> new ArrayList<>()).add(dataPartitionQueryParam);
}
}
@@ -410,7 +460,8 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
k,
new TTimeSlotList(
new ArrayList<>(v.timeSlotList), v.needLeftAll, v.needRightAll)));
- partitionSlotsMap.put(entry.getKey(), deviceToTimePartitionMap);
+ partitionSlotsMap.put(
+ PathUtils.qualifyDatabaseName(entry.getKey()), deviceToTimePartitionMap);
}
return new TDataPartitionReq(partitionSlotsMap);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java
index 70cff5c0779..05c3638a95d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.partition.SchemaPartition;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.tsfile.read.common.block.TsBlock;
@@ -49,6 +50,16 @@ public interface IAnalysis {
String getStatementType();
+ void setFinishQueryAfterAnalyze(boolean b);
+
+ void setFailStatus(TSStatus status);
+
+ boolean isFinishQueryAfterAnalyze();
+
+ default void setRealStatement(Statement realStatement) {}
+
+ void setDataPartitionInfo(DataPartition dataPartition);
+
SchemaPartition getSchemaPartitionInfo();
void setSchemaPartitionInfo(SchemaPartition schemaPartition);
@@ -60,4 +71,8 @@ public interface IAnalysis {
void addEndPointToRedirectNodeList(TEndPoint endPoint);
TimePredicate getCovertedTimePredicate();
+
+ void setDatabaseName(String databaseName);
+
+ String getDatabaseName();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
index 507bbff33bb..6db979c5e66 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
@@ -215,7 +215,7 @@ public class LoadTsfileAnalyzer implements AutoCloseable {
LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed.");
// data partition will be queried in the scheduler
- analysis.setStatement(loadTsFileStatement);
+ analysis.setRealStatement(loadTsFileStatement);
return analysis;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/SelectIntoUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/SelectIntoUtils.java
index d0c3617d530..d75f0be945f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/SelectIntoUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/SelectIntoUtils.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
-import org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.tsfile.enums.TSDataType;
@@ -41,6 +40,7 @@ import java.util.regex.Matcher;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.iotdb.commons.conf.IoTDBConstant.DOUBLE_COLONS;
import static org.apache.iotdb.commons.conf.IoTDBConstant.LEVELED_PATH_TEMPLATE_PATTERN;
+import static org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor.parseNodeString;
public class SelectIntoUtils {
@@ -108,7 +108,7 @@ public class SelectIntoUtils {
resNode = matcher.replaceFirst(sourceNodes[index]);
matcher = LEVELED_PATH_TEMPLATE_PATTERN.matcher(resNode);
}
- return ASTVisitor.parseNodeString(resNode);
+ return parseNodeString(resNode);
}
public static boolean checkIsAllRawSeriesQuery(List<Expression> expressions) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
index 46d9f53ea1f..0ab29eacea7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -75,23 +76,9 @@ public class DataNodeTTLCache {
}
public long getTTL(IDeviceID deviceID) {
- try {
- // TODO Tien change this way
- return getTTL(deviceID.toString());
- } catch (IllegalPathException e) {
- return Long.MAX_VALUE;
- }
- }
-
- /** Get ttl with time precision conversion. */
- public long getTTL(String path) throws IllegalPathException {
lock.readLock().lock();
try {
- long ttl = ttlCache.getClosestTTL(PathUtils.splitPathToDetachedNodes(path));
- return ttl == Long.MAX_VALUE
- ? ttl
- : CommonDateTimeUtils.convertMilliTimeWithPrecision(
- ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
+ return ttlCache.getClosestTTL(CommonUtils.deviceIdToStringArray(deviceID));
} finally {
lock.readLock().unlock();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaAutoCreation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaAutoCreation.java
index e54e8a37d9e..1d6468583ad 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaAutoCreation.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaAutoCreation.java
@@ -39,7 +39,9 @@ public interface ISchemaAutoCreation {
TSDataType getDataType(int index);
+ // TODO: Check necessity because all implementations return null
TSEncoding getEncoding(int index);
+ // TODO: Check necessity because all implementations return null
CompressionType getCompressionType(int index);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
index 7bbdd5cefc6..aabfc722bbd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
@@ -24,6 +24,9 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement;
@@ -32,11 +35,15 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
public class SchemaValidator {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SchemaValidator.class);
+
public static void validate(
ISchemaFetcher schemaFetcher, InsertBaseStatement insertStatement, MPPQueryContext context) {
try {
@@ -55,6 +62,25 @@ public class SchemaValidator {
}
}
+ public static void validate(
+ Metadata metadata, WrappedInsertStatement insertStatement, MPPQueryContext context) {
+ try {
+ String databaseName = context.getSession().getDatabaseName().orElse(null);
+ final TableSchema incomingSchema = insertStatement.getTableSchema();
+ final TableSchema realSchema =
+ metadata.validateTableHeaderSchema(databaseName, incomingSchema, context).orElse(null);
+ if (realSchema == null) {
+ throw new SemanticException(
+ "Schema validation failed, table cannot be created: " + incomingSchema);
+ }
+ insertStatement.validateTableSchema(realSchema);
+ insertStatement.validateDeviceSchema(metadata, context);
+ insertStatement.updateAfterSchemaValidation(context);
+ } catch (QueryProcessException e) {
+ throw new SemanticException(e.getMessage());
+ }
+ }
+
public static ISchemaTree validate(
ISchemaFetcher schemaFetcher,
List<PartialPath> devicePaths,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
index d2024cadc39..62cdf8c2810 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDataNodesTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask;
+import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor;
@@ -48,6 +49,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property;
@@ -61,6 +63,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use;
import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.tsfile.enums.TSDataType;
@@ -231,4 +234,10 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
protected IConfigTask visitCurrentDatabase(CurrentDatabase node, MPPQueryContext context) {
return super.visitCurrentDatabase(node, context);
}
+
+ @Override
+ protected IConfigTask visitFlush(Flush node, MPPQueryContext context) {
+ context.setQueryType(QueryType.WRITE);
+ return new FlushTask(((FlushStatement) node.getInnerTreeStatement()));
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 71f4f12df8e..643f1e46992 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -2277,6 +2277,7 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
createLogicalViewStatement.setQueryStatement(alterLogicalViewStatement.getQueryStatement());
final Analysis analysis = Analyzer.analyze(createLogicalViewStatement, context);
+ analysis.setDatabaseName(context.getSession().getDatabaseName().orElse(null));
if (analysis.isFailed()) {
future.setException(
new IoTDBException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java
index 726bd0c75db..f786e753023 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema;
import org.apache.iotdb.commons.schema.table.column.IdColumnSchema;
import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
@@ -52,6 +53,7 @@ public class AlterTableAddColumnTask implements IConfigTask {
public AlterTableAddColumnTask(
String database, String tableName, List<ColumnSchema> columnList, String queryId) {
+ database = PathUtils.qualifyDatabaseName(database);
this.database = database;
this.tableName = tableName;
this.columnList = parseInputColumnSchema(columnList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/CreateTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/CreateTableTask.java
index c0e62f4a183..51a7015525d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/CreateTableTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/CreateTableTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational;
import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
@@ -35,6 +36,7 @@ public class CreateTableTask implements IConfigTask {
private final boolean ifNotExists;
public CreateTableTask(TsTable table, String database, boolean ifNotExists) {
+ database = PathUtils.qualifyDatabaseName(database);
this.table = table;
this.database = database;
this.ifNotExists = ifNotExists;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/StatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/StatementMemorySourceVisitor.java
index 20bce13a41b..c19bd14b6de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/StatementMemorySourceVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/StatementMemorySourceVisitor.java
@@ -79,7 +79,7 @@ public class StatementMemorySourceVisitor
@Override
public StatementMemorySource visitExplain(
ExplainStatement node, StatementMemorySourceContext context) {
- context.getAnalysis().setStatement(node.getQueryStatement());
+ context.getAnalysis().setRealStatement(node.getQueryStatement());
DatasetHeader header =
new DatasetHeader(
Collections.singletonList(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDown.java
index 1205451ea25..0ed7fadf3c4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDown.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDown.java
@@ -80,7 +80,7 @@ public class AggregationPushDown implements PlanOptimizer {
@Override
public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) {
- if (analysis.getStatement().getType() != StatementType.QUERY) {
+ if (analysis.getTreeStatement().getType() != StatementType.QUERY) {
return plan;
}
QueryStatement queryStatement = analysis.getQueryStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/ColumnInjectionPushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/ColumnInjectionPushDown.java
index f90346f3c51..921ab330588 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/ColumnInjectionPushDown.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/ColumnInjectionPushDown.java
@@ -63,7 +63,7 @@ public class ColumnInjectionPushDown implements PlanOptimizer {
@Override
public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) {
- if (analysis.getStatement().getType() != StatementType.QUERY) {
+ if (analysis.getTreeStatement().getType() != StatementType.QUERY) {
return plan;
}
QueryStatement queryStatement = analysis.getQueryStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
index df73a23dd18..da560a63026 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
@@ -67,7 +67,7 @@ public class LimitOffsetPushDown implements PlanOptimizer {
@Override
public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) {
- if (analysis.getStatement().getType() != StatementType.QUERY) {
+ if (analysis.getTreeStatement().getType() != StatementType.QUERY) {
return plan;
}
QueryStatement queryStatement = analysis.getQueryStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/OrderByExpressionWithLimitChangeToTopK.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/OrderByExpressionWithLimitChangeToTopK.java
index 1f0730b3c6b..90b5d2fdb43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/OrderByExpressionWithLimitChangeToTopK.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/OrderByExpressionWithLimitChangeToTopK.java
@@ -65,7 +65,7 @@ public class OrderByExpressionWithLimitChangeToTopK implements PlanOptimizer {
@Override
public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) {
- if (analysis.getStatement().getType() != StatementType.QUERY) {
+ if (analysis.getTreeStatement().getType() != StatementType.QUERY) {
return plan;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDown.java
index ff1d4a07ea7..ed15f82ea4b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDown.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDown.java
@@ -62,7 +62,7 @@ public class PredicatePushDown implements PlanOptimizer {
@Override
public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) {
- if (analysis.getStatement().getType() != StatementType.QUERY) {
+ if (analysis.getTreeStatement().getType() != StatementType.QUERY) {
return plan;
}
QueryStatement queryStatement = analysis.getQueryStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
index 68bef30462f..e67d5243bd7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser;
@@ -100,6 +101,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.utils.TimeDuration;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
import java.nio.ByteBuffer;
import java.time.ZoneId;
@@ -282,6 +284,22 @@ public class StatementGenerator {
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0]));
insertStatement.setAligned(insertRecordReq.isAligned);
insertStatement.fillValues(insertRecordReq.values);
+ if (insertRecordReq.isSetIsWriteToTable()) {
+ insertStatement.setWriteToTable(insertRecordReq.isIsWriteToTable());
+ if (!insertRecordReq.isSetColumnCategoryies()
+ || insertRecordReq.getColumnCategoryiesSize() != insertRecordReq.getMeasurementsSize()) {
+ throw new IllegalArgumentException(
+ "Missing or invalid column categories for table " + "insertion");
+ }
+ TsTableColumnCategory[] columnCategories =
+ new TsTableColumnCategory[insertRecordReq.getColumnCategoryies().size()];
+ for (int i = 0; i < columnCategories.length; i++) {
+ columnCategories[i] =
+ TsTableColumnCategory.fromTsFileColumnType(
+ ColumnType.values()[insertRecordReq.getColumnCategoryies().get(i).intValue()]);
+ }
+ insertStatement.setColumnCategories(columnCategories);
+ }
PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime);
return insertStatement;
}
@@ -335,6 +353,22 @@ public class StatementGenerator {
}
insertStatement.setDataTypes(dataTypes);
insertStatement.setAligned(insertTabletReq.isAligned);
+ insertStatement.setWriteToTable(insertTabletReq.isWriteToTable());
+ if (insertTabletReq.isWriteToTable()) {
+ if (!insertTabletReq.isSetColumnCategories()
+ || insertTabletReq.getColumnCategoriesSize() != insertTabletReq.getMeasurementsSize()) {
+ throw new IllegalArgumentException(
+ "Missing or invalid column categories for table " + "insertion");
+ }
+ TsTableColumnCategory[] columnCategories =
+ new TsTableColumnCategory[insertTabletReq.columnCategories.size()];
+ for (int i = 0; i < columnCategories.length; i++) {
+ columnCategories[i] =
+ TsTableColumnCategory.fromTsFileColumnType(
+ ColumnType.values()[insertTabletReq.columnCategories.get(i).intValue()]);
+ }
+ insertStatement.setColumnCategories(columnCategories);
+ }
PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime);
return insertStatement;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
index 5f460f0267d..65f28cd4a9e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
@@ -45,7 +45,8 @@ public class LogicalPlanner {
public LogicalQueryPlan plan(Analysis analysis) {
long startTime = System.nanoTime();
- PlanNode rootNode = new LogicalPlanVisitor(analysis).process(analysis.getStatement(), context);
+ PlanNode rootNode =
+ new LogicalPlanVisitor(analysis).process(analysis.getTreeStatement(), context);
// optimize the query logical plan
if (analysis.isQuery()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java
index 843374ec49c..a819fa950de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java
@@ -162,9 +162,9 @@ public class TreeModelPlanner implements IPlanner {
// Get the inner statement of PipeEnrichedStatement
Statement statementToRedirect =
- analysis.getStatement() instanceof PipeEnrichedStatement
- ? ((PipeEnrichedStatement) analysis.getStatement()).getInnerStatement()
- : analysis.getStatement();
+ analysis.getTreeStatement() instanceof PipeEnrichedStatement
+ ? ((PipeEnrichedStatement) analysis.getTreeStatement()).getInnerStatement()
+ : analysis.getTreeStatement();
if (statementToRedirect instanceof InsertBaseStatement
&& !analysis.isFinishQueryAfterAnalyze()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java
index 4a4d168a321..dd0974ac635 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java
@@ -89,7 +89,7 @@ public class DistributionPlanner {
public PlanNode addExchangeNode(PlanNode root) {
ExchangeNodeAdder adder = new ExchangeNodeAdder(this.analysis);
NodeGroupContext nodeGroupContext =
- new NodeGroupContext(context, analysis.getStatement(), root);
+ new NodeGroupContext(context, analysis.getTreeStatement(), root);
PlanNode newRoot = adder.visit(root, nodeGroupContext);
adjustUpStream(newRoot, nodeGroupContext);
return newRoot;
@@ -112,8 +112,8 @@ public class DistributionPlanner {
}
final boolean needShuffleSinkNode =
- analysis.getStatement() instanceof QueryStatement
- && needShuffleSinkNode((QueryStatement) analysis.getStatement(), context);
+ analysis.getTreeStatement() instanceof QueryStatement
+ && needShuffleSinkNode((QueryStatement) analysis.getTreeStatement(), context);
adjustUpStreamHelper(root, new HashMap<>(), needShuffleSinkNode, context);
}
@@ -183,7 +183,7 @@ public class DistributionPlanner {
}
public PlanNode optimize(PlanNode rootWithExchange) {
- if (analysis.getStatement() != null && analysis.getStatement().isQuery()) {
+ if (analysis.getTreeStatement() != null && analysis.getTreeStatement().isQuery()) {
for (PlanOptimizer optimizer : optimizers) {
rootWithExchange = optimizer.optimize(rootWithExchange, analysis, context);
}
@@ -201,7 +201,7 @@ public class DistributionPlanner {
PlanNode rootWithExchange = addExchangeNode(rootAfterRewrite);
PlanNode optimizedRootWithExchange = optimize(rootWithExchange);
- if (analysis.getStatement() != null && analysis.getStatement().isQuery()) {
+ if (analysis.getTreeStatement() != null && analysis.getTreeStatement().isQuery()) {
analysis
.getRespDatasetHeader()
.setColumnToTsBlockIndexMap(optimizedRootWithExchange.getOutputColumnNames());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
index 0a7f17166c3..e955f5f5a22 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
@@ -106,8 +106,8 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
dataNodeFIMap.get(fragmentInstance.getHostDataNode()).size()));
// compute dataNodeSeriesScanNum in LastQueryScanNode
- if (analysis.getStatement() instanceof QueryStatement
- && ((QueryStatement) analysis.getStatement()).isLastQuery()) {
+ if (analysis.getTreeStatement() instanceof QueryStatement
+ && ((QueryStatement) analysis.getTreeStatement()).isLastQuery()) {
final Map<Path, AtomicInteger> pathSumMap = new HashMap<>();
dataNodeFIMap
.values()
@@ -183,11 +183,11 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
return v;
});
- if (analysis.getStatement() instanceof QueryStatement
- || analysis.getStatement() instanceof ExplainAnalyzeStatement
- || analysis.getStatement() instanceof ShowQueriesStatement
- || (analysis.getStatement() instanceof ShowTimeSeriesStatement
- && ((ShowTimeSeriesStatement) analysis.getStatement()).isOrderByHeat())) {
+ if (analysis.getTreeStatement() instanceof QueryStatement
+ || analysis.getTreeStatement() instanceof ExplainAnalyzeStatement
+ || analysis.getTreeStatement() instanceof ShowQueriesStatement
+ || (analysis.getTreeStatement() instanceof ShowTimeSeriesStatement
+ && ((ShowTimeSeriesStatement) analysis.getTreeStatement()).isOrderByHeat())) {
fragmentInstance.getFragment().generateTypeProvider(queryContext.getTypeProvider());
}
instanceMap.putIfAbsent(fragment.getId(), fragmentInstance);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java
index b1d1fe80123..d43b18c3241 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java
@@ -31,18 +31,32 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import java.util.ArrayList;
import java.util.List;
+import java.util.function.BiFunction;
public class WriteFragmentParallelPlanner implements IFragmentParallelPlaner {
private SubPlan subPlan;
private IAnalysis analysis;
private MPPQueryContext queryContext;
+ private BiFunction<WritePlanNode, IAnalysis, List<WritePlanNode>> nodeSplitter;
public WriteFragmentParallelPlanner(
SubPlan subPlan, IAnalysis analysis, MPPQueryContext queryContext) {
this.subPlan = subPlan;
this.analysis = analysis;
this.queryContext = queryContext;
+ this.nodeSplitter = WritePlanNode::splitByPartition;
+ }
+
+ public WriteFragmentParallelPlanner(
+ SubPlan subPlan,
+ IAnalysis analysis,
+ MPPQueryContext queryContext,
+ BiFunction<WritePlanNode, IAnalysis, List<WritePlanNode>> nodeSplitter) {
+ this.subPlan = subPlan;
+ this.analysis = analysis;
+ this.queryContext = queryContext;
+ this.nodeSplitter = nodeSplitter;
}
@Override
@@ -50,9 +64,10 @@ public class WriteFragmentParallelPlanner implements IFragmentParallelPlaner {
PlanFragment fragment = subPlan.getPlanFragment();
PlanNode node = fragment.getPlanNodeTree();
if (!(node instanceof WritePlanNode)) {
- throw new IllegalArgumentException("PlanNode should be IWritePlanNode in WRITE operation");
+ throw new IllegalArgumentException(
+ "PlanNode should be IWritePlanNode in WRITE operation:" + node.getClass());
}
- List<WritePlanNode> splits = ((WritePlanNode) node).splitByPartition(analysis);
+ List<WritePlanNode> splits = nodeSplitter.apply(((WritePlanNode) node), analysis);
List<FragmentInstance> ret = new ArrayList<>();
for (WritePlanNode split : splits) {
FragmentInstance instance =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
index 91ab840732a..2f6f725b8aa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
@@ -111,6 +111,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -240,7 +243,11 @@ public enum PlanNodeType {
TABLE_MERGESORT_NODE((short) 1007),
TABLE_TOPK_NODE((short) 1008),
TABLE_COLLECT_NODE((short) 1009),
- TABLE_STREAM_SORT_NODE((short) 1010);
+ TABLE_STREAM_SORT_NODE((short) 1010),
+
+ RELATIONAL_INSERT_TABLET((short) 2000),
+ RELATIONAL_INSERT_ROW((short) 2001),
+ RELATIONAL_INSERT_ROWS((short) 2002);
public static final int BYTES = Short.BYTES;
@@ -273,6 +280,12 @@ public enum PlanNodeType {
return InsertRowsNode.deserializeFromWAL(stream);
case 44:
return DeleteDataNode.deserializeFromWAL(stream);
+ case 2000:
+ return RelationalInsertTabletNode.deserializeFromWAL(stream);
+ case 2001:
+ return RelationalInsertRowNode.deserializeFromWAL(stream);
+ case 2002:
+ return RelationalInsertRowsNode.deserializeFromWAL(stream);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
@@ -289,6 +302,12 @@ public enum PlanNodeType {
return InsertRowsNode.deserializeFromWAL(buffer);
case 44:
return DeleteDataNode.deserializeFromWAL(buffer);
+ case 2000:
+ return RelationalInsertTabletNode.deserializeFromWAL(buffer);
+ case 2001:
+ return RelationalInsertRowNode.deserializeFromWAL(buffer);
+ case 2002:
+ return RelationalInsertRowsNode.deserializeFromWAL(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
@@ -518,6 +537,12 @@ public enum PlanNodeType {
.deserialize(buffer);
case 1008:
return TopKNode.deserialize(buffer);
+ case 2000:
+ return RelationalInsertTabletNode.deserialize(buffer);
+ case 2001:
+ return RelationalInsertRowNode.deserialize(buffer);
+ case 2002:
+ return RelationalInsertRowsNode.deserialize(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index 6922d770fac..1ee8ec8b6ac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -116,6 +116,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
@@ -511,10 +514,22 @@ public abstract class PlanVisitor<R, C> {
return visitPlan(node, context);
}
+ public R visitRelationalInsertRow(RelationalInsertRowNode node, C context) {
+ return visitPlan(node, context);
+ }
+
+ public R visitRelationalInsertRows(RelationalInsertRowsNode node, C context) {
+ return visitPlan(node, context);
+ }
+
public R visitInsertTablet(InsertTabletNode node, C context) {
return visitPlan(node, context);
}
+ public R visitRelationalInsertTablet(RelationalInsertTabletNode node, C context) {
+ return visitInsertTablet(node, context);
+ }
+
public R visitInsertRows(InsertRowsNode node, C context) {
return visitPlan(node, context);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
index 540f9e4d9f6..be5de4cc819 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
@@ -92,10 +92,11 @@ public class LoadTsFileNode extends WritePlanNode {
public List<WritePlanNode> splitByPartition(IAnalysis analysis) {
List<WritePlanNode> res = new ArrayList<>();
LoadTsFileStatement statement =
- ((Analysis) analysis).getStatement() instanceof PipeEnrichedStatement
+ ((Analysis) analysis).getTreeStatement() instanceof PipeEnrichedStatement
? (LoadTsFileStatement)
- ((PipeEnrichedStatement) ((Analysis) analysis).getStatement()).getInnerStatement()
- : (LoadTsFileStatement) ((Analysis) analysis).getStatement();
+ ((PipeEnrichedStatement) ((Analysis) analysis).getTreeStatement())
+ .getInnerStatement()
+ : (LoadTsFileStatement) ((Analysis) analysis).getTreeStatement();
for (int i = 0; i < resources.size(); i++) {
res.add(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java
index f1f781fc918..77b936a7885 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java
@@ -108,7 +108,10 @@ public class TableDeviceQueryNode extends TableDeviceSourceNode {
}
}
- Expression.serialize(idFuzzyPredicate, byteBuffer);
+ ReadWriteIOUtils.write(idFuzzyPredicate == null ? (byte) 0 : (byte) 1, byteBuffer);
+ if (idFuzzyPredicate != null) {
+ Expression.serialize(idFuzzyPredicate, byteBuffer);
+ }
ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer);
for (ColumnHeader columnHeader : columnHeaderList) {
@@ -130,7 +133,10 @@ public class TableDeviceQueryNode extends TableDeviceSourceNode {
}
}
- Expression.serialize(idFuzzyPredicate, stream);
+ ReadWriteIOUtils.write(idFuzzyPredicate == null ? (byte) 0 : (byte) 1, stream);
+ if (idFuzzyPredicate != null) {
+ Expression.serialize(idFuzzyPredicate, stream);
+ }
ReadWriteIOUtils.write(columnHeaderList.size(), stream);
for (ColumnHeader columnHeader : columnHeaderList) {
@@ -152,7 +158,10 @@ public class TableDeviceQueryNode extends TableDeviceSourceNode {
}
}
- Expression idFuzzyFilter = Expression.deserialize(buffer);
+ Expression idFuzzyFilter = null;
+ if (buffer.get() == 1) {
+ idFuzzyFilter = Expression.deserialize(buffer);
+ }
size = ReadWriteIOUtils.readInt(buffer);
List<ColumnHeader> columnHeaderList = new ArrayList<>(size);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java
index 430ab19c7c9..802bd579fdf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java
@@ -168,11 +168,6 @@ public class InsertMultiTabletsNode extends InsertNode {
return StatusUtils.getFailingStatus(results, insertTabletNodeList.size());
}
- @Override
- public List<PlanNode> getChildren() {
- return null;
- }
-
@Override
public void addChild(PlanNode child) {}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
index 161b6ac9d2e..f18de9ef2b9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
@@ -23,10 +23,12 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.commons.consensus.index.ComparableConsensusRequest;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
@@ -41,7 +43,10 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Objects;
public abstract class InsertNode extends SearchNode implements ComparableConsensusRequest {
@@ -59,6 +64,10 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
protected String[] measurements;
protected TSDataType[] dataTypes;
+ protected TsTableColumnCategory[] columnCategories;
+ protected List<Integer> idColumnIndices;
+ protected int measurementColumnCnt = -1;
+
protected int failedMeasurementNumber = 0;
/**
@@ -86,11 +95,22 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
boolean isAligned,
String[] measurements,
TSDataType[] dataTypes) {
+ this(id, devicePath, isAligned, measurements, dataTypes, null);
+ }
+
+ protected InsertNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ TsTableColumnCategory[] columnCategories) {
super(id);
this.devicePath = devicePath;
this.isAligned = isAligned;
this.measurements = measurements;
this.dataTypes = dataTypes;
+ setColumnCategories(columnCategories);
}
public TRegionReplicaSet getDataRegionReplicaSet() {
@@ -129,6 +149,22 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
return measurements;
}
+ public int measureColumnCnt() {
+ if (columnCategories == null) {
+ return measurements.length;
+ }
+ return (int)
+ Arrays.stream(columnCategories)
+ .filter(col -> col == TsTableColumnCategory.MEASUREMENT)
+ .count();
+ }
+
+ public boolean isValidMeasurement(int i) {
+ return measurementSchemas != null
+ && measurementSchemas[i] != null
+ && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.MEASUREMENT);
+ }
+
public void setMeasurements(String[] measurements) {
this.measurements = measurements;
}
@@ -137,6 +173,20 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
return dataTypes;
}
+ public int getMeasurementColumnCnt() {
+ if (measurementColumnCnt == -1) {
+ measurementColumnCnt = 0;
+ if (measurementSchemas != null) {
+ for (int i = 0; i < measurementSchemas.length; i++) {
+ if (isValidMeasurement(i)) {
+ measurementColumnCnt++;
+ }
+ }
+ }
+ }
+ return measurementColumnCnt;
+ }
+
public TSDataType getDataType(int index) {
return dataTypes[index];
}
@@ -185,6 +235,7 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
}
// region Serialization methods for WAL
+
/** Serialized size of measurement schemas, ignoring failed time series */
protected int serializeMeasurementSchemasSize() {
int byteLen = 0;
@@ -284,9 +335,15 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
InsertNode that = (InsertNode) o;
return isAligned == that.isAligned
&& Objects.equals(devicePath, that.devicePath)
@@ -306,4 +363,29 @@ public abstract class InsertNode extends SearchNode implements ComparableConsens
result = 31 * result + Arrays.hashCode(dataTypes);
return result;
}
+
+ public TsTableColumnCategory[] getColumnCategories() {
+ return columnCategories;
+ }
+
+ public void setColumnCategories(TsTableColumnCategory[] columnCategories) {
+ this.columnCategories = columnCategories;
+ if (columnCategories != null) {
+ idColumnIndices = new ArrayList<>();
+ for (int i = 0; i < columnCategories.length; i++) {
+ if (columnCategories[i].equals(TsTableColumnCategory.ID)) {
+ idColumnIndices.add(i);
+ }
+ }
+ }
+ }
+
+ public String getTableName() {
+ return null;
+ }
+
+ @Override
+ public List<PlanNode> getChildren() {
+ return Collections.emptyList();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
index 3f70e312223..464e807ebab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
@@ -58,11 +58,12 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
private static final byte TYPE_RAW_STRING = -1;
- private static final byte TYPE_NULL = -2;
+ private static final byte TYPE_NULL_WITHOUT_TYPE = -2;
+ private static final byte TYPE_NULL_WITH_TYPE = -3;
private static final String UNSUPPORTED_DATA_TYPE = "Unsupported data type: ";
- private static final String DESERIALIZE_ERROR = "Cannot deserialize InsertRowNode";
+ protected static final String DESERIALIZE_ERROR = "Cannot deserialize InsertRowNode";
private long time;
private Object[] values;
@@ -113,7 +114,7 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
analysis
.getDataPartitionInfo()
.getDataRegionReplicaSetForWriting(
- devicePath.getIDeviceIDAsFullDevice(), timePartitionSlot);
+ getDeviceID(), timePartitionSlot, analysis.getDatabaseName());
// collect redirectInfo
analysis.setRedirectNodeList(
Collections.singletonList(
@@ -121,11 +122,6 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
return Collections.singletonList(this);
}
- @Override
- public List<PlanNode> getChildren() {
- return Collections.emptyList();
- }
-
@Override
public void addChild(PlanNode child) {
// no child for InsertRowNode
@@ -141,6 +137,11 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
throw new NotImplementedException("clone of Insert is not implemented");
}
+ @Override
+ public String toString() {
+ return "InsertRowNode{" + "time=" + time + ", values=" + Arrays.toString(values) + '}';
+ }
+
@Override
public int allowedChildCount() {
return NO_CHILD_ALLOWED;
@@ -214,13 +215,13 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
@Override
protected void serializeAttributes(ByteBuffer byteBuffer) {
- PlanNodeType.INSERT_ROW.serialize(byteBuffer);
+ getType().serialize(byteBuffer);
subSerialize(byteBuffer);
}
@Override
protected void serializeAttributes(DataOutputStream stream) throws IOException {
- PlanNodeType.INSERT_ROW.serialize(stream);
+ getType().serialize(stream);
subSerialize(stream);
}
@@ -312,7 +313,11 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
}
// serialize null value
if (values[i] == null) {
- ReadWriteIOUtils.write(TYPE_NULL, buffer);
+ ReadWriteIOUtils.write(
+ dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer);
+ if (dataTypes[i] != null) {
+ ReadWriteIOUtils.write(dataTypes[i], buffer);
+ }
continue;
}
// types are not determined, the situation mainly occurs when the plan uses string values
@@ -367,7 +372,11 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
}
// serialize null value
if (values[i] == null) {
- ReadWriteIOUtils.write(TYPE_NULL, stream);
+ ReadWriteIOUtils.write(
+ dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, stream);
+ if (dataTypes[i] != null) {
+ ReadWriteIOUtils.write(dataTypes[i], stream);
+ }
continue;
}
// types are not determined, the situation mainly occurs when the plan uses string values
@@ -463,8 +472,13 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
// types are not determined, the situation mainly occurs when the node uses string values
// and is forwarded to other nodes
byte typeNum = (byte) ReadWriteIOUtils.read(buffer);
- if (typeNum == TYPE_RAW_STRING || typeNum == TYPE_NULL) {
+ if (typeNum == TYPE_RAW_STRING
+ || typeNum == TYPE_NULL_WITHOUT_TYPE
+ || typeNum == TYPE_NULL_WITH_TYPE) {
values[i] = typeNum == TYPE_RAW_STRING ? ReadWriteIOUtils.readString(buffer) : null;
+ if (typeNum == TYPE_NULL_WITH_TYPE) {
+ dataTypes[i] = ReadWriteIOUtils.readDataType(buffer);
+ }
continue;
}
dataTypes[i] = TSDataType.values()[typeNum];
@@ -531,6 +545,9 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
// serialize null value
if (values[i] == null) {
size += Byte.BYTES;
+ if (dataTypes[i] != null) {
+ size += Byte.BYTES;
+ }
continue;
}
size += Byte.BYTES;
@@ -572,7 +589,7 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
*/
@Override
public void serializeToWAL(IWALByteBufferView buffer) {
- buffer.putShort(PlanNodeType.INSERT_ROW.getNodeType());
+ buffer.putShort(getType().getNodeType());
buffer.putLong(searchIndex);
subSerialize(buffer);
}
@@ -605,7 +622,11 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
}
// serialize null value
if (values[i] == null) {
- WALWriteUtils.write(TYPE_NULL, buffer);
+ WALWriteUtils.write(
+ dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer);
+ if (dataTypes[i] != null) {
+ WALWriteUtils.write(dataTypes[i], buffer);
+ }
continue;
}
WALWriteUtils.write(dataTypes[i], buffer);
@@ -692,7 +713,13 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
public void fillDataTypesAndValuesFromWAL(DataInputStream stream) throws IOException {
for (int i = 0; i < dataTypes.length; i++) {
byte typeNum = stream.readByte();
- if (typeNum == TYPE_NULL) {
+ if (typeNum == TYPE_RAW_STRING
+ || typeNum == TYPE_NULL_WITHOUT_TYPE
+ || typeNum == TYPE_NULL_WITH_TYPE) {
+ values[i] = typeNum == TYPE_RAW_STRING ? ReadWriteIOUtils.readString(stream) : null;
+ if (typeNum == TYPE_NULL_WITH_TYPE) {
+ dataTypes[i] = ReadWriteIOUtils.readDataType(stream);
+ }
continue;
}
dataTypes[i] = TSDataType.values()[typeNum];
@@ -778,7 +805,13 @@ public class InsertRowNode extends InsertNode implements WALEntryValue {
public void fillDataTypesAndValuesFromWAL(ByteBuffer buffer) {
for (int i = 0; i < dataTypes.length; i++) {
byte typeNum = buffer.get();
- if (typeNum == TYPE_NULL) {
+ if (typeNum == TYPE_RAW_STRING
+ || typeNum == TYPE_NULL_WITHOUT_TYPE
+ || typeNum == TYPE_NULL_WITH_TYPE) {
+ values[i] = typeNum == TYPE_RAW_STRING ? ReadWriteIOUtils.readString(buffer) : null;
+ if (typeNum == TYPE_NULL_WITH_TYPE) {
+ dataTypes[i] = ReadWriteIOUtils.readDataType(buffer);
+ }
continue;
}
dataTypes[i] = TSDataType.values()[typeNum];
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java
index ce08eccf7dc..5333deca501 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java
@@ -120,11 +120,6 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
return StatusUtils.getFailingStatus(results, insertRowNodeList.size());
}
- @Override
- public List<PlanNode> getChildren() {
- return Collections.emptyList();
- }
-
@Override
public void addChild(PlanNode child) {
// Do nothing
@@ -145,6 +140,16 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
&& Objects.equals(insertRowNodeList, that.insertRowNodeList);
}
+ @Override
+ public String toString() {
+ return "InsertRowsNode{"
+ + "insertRowNodeIndexList="
+ + insertRowNodeIndexList
+ + ", insertRowNodeList="
+ + insertRowNodeList
+ + '}';
+ }
+
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), insertRowNodeIndexList, insertRowNodeList);
@@ -193,7 +198,7 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
@Override
protected void serializeAttributes(ByteBuffer byteBuffer) {
- PlanNodeType.INSERT_ROWS.serialize(byteBuffer);
+ getType().serialize(byteBuffer);
ReadWriteIOUtils.write(insertRowNodeList.size(), byteBuffer);
@@ -207,7 +212,7 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
@Override
protected void serializeAttributes(DataOutputStream stream) throws IOException {
- PlanNodeType.INSERT_ROWS.serialize(stream);
+ getType().serialize(stream);
ReadWriteIOUtils.write(insertRowNodeList.size(), stream);
@@ -238,12 +243,14 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
for (int i = 0; i < insertRowNodeList.size(); i++) {
InsertRowNode insertRowNode = insertRowNodeList.get(i);
// Data region for insert row node
+ // each row may belong to different database, pass null for auto-detection
TRegionReplicaSet dataRegionReplicaSet =
analysis
.getDataPartitionInfo()
.getDataRegionReplicaSetForWriting(
insertRowNode.devicePath.getIDeviceIDAsFullDevice(),
- TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()));
+ TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()),
+ null);
// Collect redirectInfo
redirectInfo.add(dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint());
InsertRowsNode tmpNode = splitMap.get(dataRegionReplicaSet);
@@ -312,7 +319,7 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue {
*/
@Override
public void serializeToWAL(IWALByteBufferView buffer) {
- buffer.putShort(PlanNodeType.INSERT_ROWS.getNodeType());
+ buffer.putShort(getType().getNodeType());
buffer.putLong(searchIndex);
subSerialize(buffer);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java
index 3c2797c5077..c9de66c7aa8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java
@@ -126,11 +126,6 @@ public class InsertRowsOfOneDeviceNode extends InsertNode {
storeMeasurementsAndDataType();
}
- @Override
- public List<PlanNode> getChildren() {
- return null;
- }
-
@Override
public void addChild(PlanNode child) {}
@@ -170,7 +165,9 @@ public class InsertRowsOfOneDeviceNode extends InsertNode {
analysis
.getDataPartitionInfo()
.getDataRegionReplicaSetForWriting(
- devicePath.getIDeviceIDAsFullDevice(), timePartitionSlot);
+ devicePath.getIDeviceIDAsFullDevice(),
+ timePartitionSlot,
+ analysis.getDatabaseName());
Map<TTimePartitionSlot, List<InsertRowNode>> tmpMap =
splitMap.computeIfAbsent(dataRegionReplicaSet, k -> new HashMap<>());
Map<TTimePartitionSlot, List<Integer>> tmpIndexMap =
@@ -307,9 +304,15 @@ public class InsertRowsOfOneDeviceNode extends InsertNode {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
InsertRowsOfOneDeviceNode that = (InsertRowsOfOneDeviceNode) o;
return Objects.equals(insertRowNodeIndexList, that.insertRowNodeIndexList)
&& Objects.equals(insertRowNodeList, that.insertRowNodeList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
index 14cc778a52b..be5e26f7858 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
@@ -20,11 +20,14 @@
package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
@@ -32,17 +35,23 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils;
+import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.NotImplementedException;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
@@ -58,18 +67,22 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
+import java.util.function.IntToLongFunction;
+
+import static org.apache.iotdb.db.utils.CommonUtils.isAlive;
public class InsertTabletNode extends InsertNode implements WALEntryValue {
private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported.";
- private long[] times; // times should be sorted. It is done in the session API.
+ protected long[] times; // times should be sorted. It is done in the session API.
- private BitMap[] bitMaps;
- private Object[] columns;
+ protected BitMap[] bitMaps;
+ protected Object[] columns;
- private int rowCount = 0;
+ protected int rowCount = 0;
// When this plan is sub-plan split from another InsertTabletNode, this indicates the original
// positions of values in
@@ -80,7 +93,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
// of the parent plan.
// this is usually used to back-propagate exceptions to the parent plan without losing their
// proper positions.
- private List<Integer> range;
+ protected List<Integer> range;
public InsertTabletNode(PlanNodeId id) {
super(id);
@@ -163,11 +176,6 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
this.range = range;
}
- @Override
- public List<PlanNode> getChildren() {
- return null;
- }
-
@Override
public void addChild(PlanNode child) {}
@@ -194,105 +202,154 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
@Override
public List<WritePlanNode> splitByPartition(IAnalysis analysis) {
// only single device in single database
- List<WritePlanNode> result = new ArrayList<>();
if (times.length == 0) {
return Collections.emptyList();
}
+
+ final Map<IDeviceID, PartitionSplitInfo> deviceIDSplitInfoMap = collectSplitRanges();
+ final Map<TRegionReplicaSet, List<Integer>> splitMap =
+ splitByReplicaSet(deviceIDSplitInfoMap, analysis);
+
+ return doSplit(splitMap);
+ }
+
+ private Map<IDeviceID, PartitionSplitInfo> collectSplitRanges() {
long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]);
TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]);
int startLoc = 0; // included
+ IDeviceID currDeviceId = getDeviceID(0);
+
+ Map<IDeviceID, PartitionSplitInfo> deviceIDSplitInfoMap = new HashMap<>();
- List<TTimePartitionSlot> timePartitionSlots = new ArrayList<>();
- // for each List in split, they are range1.start, range1.end, range2.start, range2.end, ...
- List<Integer> ranges = new ArrayList<>();
for (int i = 1; i < times.length; i++) { // times are sorted in session API.
- if (times[i] >= upperBoundOfTimePartition) {
+ IDeviceID nextDeviceId = getDeviceID(i);
+ if (times[i] >= upperBoundOfTimePartition || !currDeviceId.equals(nextDeviceId)) {
+ final PartitionSplitInfo splitInfo =
+ deviceIDSplitInfoMap.computeIfAbsent(
+ currDeviceId, deviceID1 -> new PartitionSplitInfo());
// a new range.
- ranges.add(startLoc); // included
- ranges.add(i); // excluded
- timePartitionSlots.add(timePartitionSlot);
+ splitInfo.ranges.add(startLoc); // included
+ splitInfo.ranges.add(i); // excluded
+ splitInfo.timePartitionSlots.add(timePartitionSlot);
// next init
startLoc = i;
upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]);
timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]);
+ currDeviceId = nextDeviceId;
}
}
+ PartitionSplitInfo splitInfo =
+ deviceIDSplitInfoMap.computeIfAbsent(currDeviceId, deviceID1 -> new PartitionSplitInfo());
// the final range
- ranges.add(startLoc); // included
- ranges.add(times.length); // excluded
- timePartitionSlots.add(timePartitionSlot);
-
- // data region for each time partition
- List<TRegionReplicaSet> dataRegionReplicaSets =
- analysis
- .getDataPartitionInfo()
- .getDataRegionReplicaSetForWriting(
- devicePath.getIDeviceIDAsFullDevice(), timePartitionSlots);
-
- // collect redirectInfo
- analysis.addEndPointToRedirectNodeList(
- dataRegionReplicaSets
- .get(dataRegionReplicaSets.size() - 1)
- .getDataNodeLocations()
- .get(0)
- .getClientRpcEndPoint());
+ splitInfo.ranges.add(startLoc); // included
+ splitInfo.ranges.add(times.length); // excluded
+ splitInfo.timePartitionSlots.add(timePartitionSlot);
+
+ return deviceIDSplitInfoMap;
+ }
+ private Map<TRegionReplicaSet, List<Integer>> splitByReplicaSet(
+ Map<IDeviceID, PartitionSplitInfo> deviceIDSplitInfoMap, IAnalysis analysis) {
Map<TRegionReplicaSet, List<Integer>> splitMap = new HashMap<>();
- for (int i = 0; i < dataRegionReplicaSets.size(); i++) {
- List<Integer> sub_ranges =
- splitMap.computeIfAbsent(dataRegionReplicaSets.get(i), x -> new ArrayList<>());
- sub_ranges.add(ranges.get(2 * i));
- sub_ranges.add(ranges.get(2 * i + 1));
+
+ for (Entry<IDeviceID, PartitionSplitInfo> entry : deviceIDSplitInfoMap.entrySet()) {
+ final IDeviceID deviceID = entry.getKey();
+ final PartitionSplitInfo splitInfo = entry.getValue();
+ final List<TRegionReplicaSet> replicaSets =
+ analysis
+ .getDataPartitionInfo()
+ .getDataRegionReplicaSetForWriting(
+ deviceID, splitInfo.timePartitionSlots, analysis.getDatabaseName());
+ splitInfo.replicaSets = replicaSets;
+ // collect redirectInfo
+ analysis.addEndPointToRedirectNodeList(
+ replicaSets
+ .get(replicaSets.size() - 1)
+ .getDataNodeLocations()
+ .get(0)
+ .getClientRpcEndPoint());
+ for (int i = 0; i < replicaSets.size(); i++) {
+ List<Integer> subRanges =
+ splitMap.computeIfAbsent(replicaSets.get(i), x -> new ArrayList<>());
+ subRanges.add(splitInfo.ranges.get(2 * i));
+ subRanges.add(splitInfo.ranges.get(2 * i + 1));
+ }
}
+ return splitMap;
+ }
- List<Integer> locs;
- for (Map.Entry<TRegionReplicaSet, List<Integer>> entry : splitMap.entrySet()) {
- // generate a new times and values
- locs = entry.getValue();
- // Avoid using system arraycopy when there is no need to split
- if (splitMap.size() == 1 && locs.size() == 2) {
- setRange(locs);
+ private List<WritePlanNode> doSplit(Map<TRegionReplicaSet, List<Integer>> splitMap) {
+ List<WritePlanNode> result = new ArrayList<>();
+
+ if (splitMap.size() == 1) {
+ final Entry<TRegionReplicaSet, List<Integer>> entry = splitMap.entrySet().iterator().next();
+ if (entry.getValue().size() == 2) {
+ // Avoid using system arraycopy when there is no need to split
+ setRange(entry.getValue());
setDataRegionReplicaSet(entry.getKey());
result.add(this);
return result;
}
- for (int i = 0; i < locs.size(); i += 2) {
- int start = locs.get(i);
- int end = locs.get(i + 1);
- int count = end - start;
- long[] subTimes = new long[count];
- int destLoc = 0;
- Object[] values = initTabletValues(dataTypes.length, count, dataTypes);
- BitMap[] bitMaps = this.bitMaps == null ? null : initBitmaps(dataTypes.length, count);
- System.arraycopy(times, start, subTimes, destLoc, end - start);
- for (int k = 0; k < values.length; k++) {
- if (dataTypes[k] != null) {
- System.arraycopy(columns[k], start, values[k], destLoc, end - start);
- }
- if (bitMaps != null && this.bitMaps[k] != null) {
- BitMap.copyOfRange(this.bitMaps[k], start, bitMaps[k], destLoc, end - start);
- }
+ }
+
+ for (Map.Entry<TRegionReplicaSet, List<Integer>> entry : splitMap.entrySet()) {
+ result.add(generateOneSplit(entry));
+ }
+ return result;
+ }
+
+ protected InsertTabletNode getEmptySplit(int count) {
+ long[] subTimes = new long[count];
+ Object[] values = initTabletValues(dataTypes.length, count, dataTypes);
+ BitMap[] newBitMaps = this.bitMaps == null ? null : initBitmaps(dataTypes.length, count);
+ return new InsertTabletNode(
+ getPlanNodeId(),
+ devicePath,
+ isAligned,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ subTimes,
+ newBitMaps,
+ values,
+ subTimes.length);
+ }
+
+ private WritePlanNode generateOneSplit(Map.Entry<TRegionReplicaSet, List<Integer>> entry) {
+ List<Integer> locs;
+ // generate a new times and values
+ locs = entry.getValue();
+ int count = 0;
+ for (int i = 0; i < locs.size(); i += 2) {
+ int start = locs.get(i);
+ int end = locs.get(i + 1);
+ count += end - start;
+ }
+
+ final InsertTabletNode subNode = getEmptySplit(count);
+ int destLoc = 0;
+
+ for (int i = 0; i < locs.size(); i += 2) {
+ int start = locs.get(i);
+ int end = locs.get(i + 1);
+ final int length = end - start;
+
+ System.arraycopy(times, start, subNode.times, destLoc, length);
+ for (int k = 0; k < subNode.columns.length; k++) {
+ if (dataTypes[k] != null) {
+ System.arraycopy(columns[k], start, subNode.columns[k], destLoc, length);
+ }
+ if (subNode.bitMaps != null && this.bitMaps[k] != null) {
+ BitMap.copyOfRange(this.bitMaps[k], start, subNode.bitMaps[k], destLoc, length);
}
- InsertTabletNode subNode =
- new InsertTabletNode(
- getPlanNodeId(),
- devicePath,
- isAligned,
- measurements,
- dataTypes,
- measurementSchemas,
- subTimes,
- bitMaps,
- values,
- subTimes.length);
- subNode.setFailedMeasurementNumber(getFailedMeasurementNumber());
- subNode.setRange(locs);
- subNode.setDataRegionReplicaSet(entry.getKey());
- result.add(subNode);
}
+ destLoc += length;
}
- return result;
+ subNode.setFailedMeasurementNumber(getFailedMeasurementNumber());
+ subNode.setRange(locs);
+ subNode.setDataRegionReplicaSet(entry.getKey());
+ return subNode;
}
@TestOnly
@@ -312,7 +369,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
return result;
}
- private Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] dataTypes) {
+ protected Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] dataTypes) {
Object[] values = new Object[columnSize];
for (int i = 0; i < values.length; i++) {
if (dataTypes[i] != null) {
@@ -344,7 +401,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
return values;
}
- private BitMap[] initBitmaps(int columnSize, int rowSize) {
+ protected BitMap[] initBitmaps(int columnSize, int rowSize) {
BitMap[] bitMaps = new BitMap[columnSize];
for (int i = 0; i < columnSize; i++) {
bitMaps[i] = new BitMap(rowSize);
@@ -369,13 +426,13 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
@Override
protected void serializeAttributes(ByteBuffer byteBuffer) {
- PlanNodeType.INSERT_TABLET.serialize(byteBuffer);
+ getType().serialize(byteBuffer);
subSerialize(byteBuffer);
}
@Override
protected void serializeAttributes(DataOutputStream stream) throws IOException {
- PlanNodeType.INSERT_TABLET.serialize(stream);
+ getType().serialize(stream);
subSerialize(stream);
}
@@ -617,12 +674,16 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
ReadWriteIOUtils.write(BytesUtils.boolToByte(boolValues[j]), stream);
}
break;
+ case STRING:
case TEXT:
case BLOB:
- case STRING:
Binary[] binaryValues = (Binary[]) column;
for (int j = 0; j < rowCount; j++) {
- ReadWriteIOUtils.write(binaryValues[j], stream);
+ if (binaryValues[j] != null) {
+ ReadWriteIOUtils.write(binaryValues[j], stream);
+ } else {
+ ReadWriteIOUtils.write(0, stream);
+ }
}
break;
default:
@@ -682,6 +743,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
// region serialize & deserialize methods for WAL
+
/** Serialized size for wal */
@Override
public int serializedSize() {
@@ -727,8 +789,10 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
size += getColumnSize(dataTypes[i], columns[i], start, end);
}
}
-
+ // isAlign
size += Byte.BYTES;
+ // column category
+
return size;
}
@@ -774,7 +838,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
public void serializeToWAL(IWALByteBufferView buffer, int start, int end) {
- buffer.putShort(PlanNodeType.INSERT_TABLET.getNodeType());
+ buffer.putShort(getType().getNodeType());
subSerialize(buffer, start, end);
}
@@ -790,12 +854,12 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
/** Serialize measurement schemas, ignoring failed time series */
- private void writeMeasurementSchemas(IWALByteBufferView buffer) {
+ protected void writeMeasurementSchemas(IWALByteBufferView buffer) {
buffer.putInt(measurements.length - getFailedMeasurementNumber());
serializeMeasurementSchemasToWAL(buffer);
}
- private void writeTimes(IWALByteBufferView buffer, int start, int end) {
+ protected void writeTimes(IWALByteBufferView buffer, int start, int end) {
buffer.putInt(end - start);
for (int i = start; i < end; i++) {
buffer.putLong(times[i]);
@@ -803,7 +867,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
/** Serialize bitmaps, ignoring failed time series */
- private void writeBitMaps(IWALByteBufferView buffer, int start, int end) {
+ protected void writeBitMaps(IWALByteBufferView buffer, int start, int end) {
buffer.put(BytesUtils.boolToByte(bitMaps != null));
if (bitMaps != null) {
for (int i = 0; i < bitMaps.length; i++) {
@@ -826,7 +890,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
/** Serialize values, ignoring failed time series */
- private void writeValues(IWALByteBufferView buffer, int start, int end) {
+ protected void writeValues(IWALByteBufferView buffer, int start, int end) {
for (int i = 0; i < columns.length; i++) {
// ignore failed partial insert
if (measurements[i] == null) {
@@ -871,13 +935,17 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
buffer.put(BytesUtils.boolToByte(boolValues[j]));
}
break;
+ case STRING:
case TEXT:
case BLOB:
- case STRING:
Binary[] binaryValues = (Binary[]) column;
for (int j = start; j < end; j++) {
- buffer.putInt(binaryValues[j].getLength());
- buffer.put(binaryValues[j].getValues());
+ if (binaryValues[j] != null) {
+ buffer.putInt(binaryValues[j].getLength());
+ buffer.put(binaryValues[j].getValues());
+ } else {
+ buffer.putInt(0);
+ }
}
break;
default:
@@ -893,7 +961,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
return insertNode;
}
- private void subDeserializeFromWAL(DataInputStream stream) throws IOException {
+ protected void subDeserializeFromWAL(DataInputStream stream) throws IOException {
searchIndex = stream.readLong();
try {
devicePath =
@@ -929,7 +997,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
return insertNode;
}
- private void subDeserializeFromWAL(ByteBuffer buffer) {
+ protected void subDeserializeFromWAL(ByteBuffer buffer) {
searchIndex = buffer.getLong();
try {
devicePath =
@@ -977,9 +1045,15 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
InsertTabletNode that = (InsertTabletNode) o;
return rowCount == that.rowCount
&& Arrays.equals(times, that.times)
@@ -1107,4 +1181,82 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
}
return new TimeValuePair(times[lastIdx], value);
}
+
+ public IDeviceID getDeviceID(int rowIdx) {
+ if (deviceID != null) {
+ return deviceID;
+ }
+ deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
+ return deviceID;
+ }
+
+ private static class PartitionSplitInfo {
+
+ // for each List in split, they are range1.start, range1.end, range2.start, range2.end, ...
+ private List<Integer> ranges = new ArrayList<>();
+ private List<TTimePartitionSlot> timePartitionSlots = new ArrayList<>();
+ private List<TRegionReplicaSet> replicaSets;
+ }
+
+ /**
+ * Split the tablet of the given range according to Table deviceID.
+ *
+ * @param start inclusive
+ * @param end exclusive
+ * @return each the number in the pair is the end offset of a device
+ */
+ public List<Pair<IDeviceID, Integer>> splitByDevice(int start, int end) {
+ return Collections.singletonList(new Pair<>(getDeviceID(), end));
+ }
+
+ /**
+ * @param results insertion result of each row
+ * @param rowTTLGetter the ttl associated with each row
+ * @return the position of the first alive row
+ * @throws OutOfTTLException if all rows have expired the TTL
+ */
+ public int checkTTL(TSStatus[] results, IntToLongFunction rowTTLGetter) throws OutOfTTLException {
+ return checkTTLInternal(results, rowTTLGetter, true);
+ }
+
+ protected int checkTTLInternal(
+ TSStatus[] results, IntToLongFunction rowTTLGetter, boolean breakOnFirstAlive)
+ throws OutOfTTLException {
+
+ /*
+ * assume that batch has been sorted by client
+ */
+ int loc = 0;
+ long ttl = 0;
+ int firstAliveLoc = -1;
+ while (loc < getRowCount()) {
+ ttl = rowTTLGetter.applyAsLong(loc);
+ long currTime = getTimes()[loc];
+ // skip points that do not satisfy TTL
+ if (!isAlive(currTime, ttl)) {
+ results[loc] =
+ RpcUtils.getStatus(
+ TSStatusCode.OUT_OF_TTL,
+ String.format(
+ "Insertion time [%s] is less than ttl time bound [%s]",
+ DateTimeUtils.convertLongToDate(currTime),
+ DateTimeUtils.convertLongToDate(CommonDateTimeUtils.currentTime() - ttl)));
+ } else {
+ if (firstAliveLoc == -1) {
+ firstAliveLoc = loc;
+ }
+ if (breakOnFirstAlive) {
+ break;
+ }
+ }
+ loc++;
+ }
+
+ if (firstAliveLoc == -1) {
+ // no alive data
+ throw new OutOfTTLException(
+ getTimes()[getTimes().length - 1], (CommonDateTimeUtils.currentTime() - ttl));
+ }
+ return firstAliveLoc;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
new file mode 100644
index 00000000000..97dd3d3673e
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class RelationalInsertRowNode extends InsertRowNode {
+ // deviceId cache for Table-view insertion
+ private IDeviceID deviceID;
+
+ public RelationalInsertRowNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ long time,
+ Object[] values,
+ boolean isNeedInferType,
+ TsTableColumnCategory[] columnCategories) {
+ super(id, devicePath, isAligned, measurements, dataTypes, time, values, isNeedInferType);
+ setColumnCategories(columnCategories);
+ }
+
+ public RelationalInsertRowNode(PlanNodeId id) {
+ super(id);
+ }
+
+ public RelationalInsertRowNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ MeasurementSchema[] measurementSchemas,
+ long time,
+ Object[] values,
+ boolean isNeedInferType,
+ TsTableColumnCategory[] columnCategories) {
+ super(
+ id,
+ devicePath,
+ isAligned,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ time,
+ values,
+ isNeedInferType);
+ setColumnCategories(columnCategories);
+ }
+
+ public IDeviceID getDeviceID() {
+ if (deviceID == null) {
+ String[] deviceIdSegments = new String[idColumnIndices.size() + 1];
+ deviceIdSegments[0] = this.getTableName();
+ for (int i = 0; i < idColumnIndices.size(); i++) {
+ final Integer columnIndex = idColumnIndices.get(i);
+ deviceIdSegments[i + 1] =
+ getValues()[columnIndex] != null ? getValues()[columnIndex].toString() : null;
+ }
+ deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+
+ return deviceID;
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitRelationalInsertRow(this, context);
+ }
+
+ public static RelationalInsertRowNode deserialize(ByteBuffer byteBuffer) {
+ RelationalInsertRowNode insertNode = new RelationalInsertRowNode(new PlanNodeId(""));
+ insertNode.subDeserialize(byteBuffer);
+ insertNode.setPlanNodeId(PlanNodeId.deserialize(byteBuffer));
+ return insertNode;
+ }
+
+ /**
+ * Deserialize from wal.
+ *
+ * @param stream - DataInputStream
+ * @return RelationalInsertRowNode
+ * @throws IOException - If an I/O error occurs.
+ * @throws IllegalArgumentException - If meets illegal argument.
+ */
+ public static RelationalInsertRowNode deserializeFromWAL(DataInputStream stream)
+ throws IOException {
+ long searchIndex = stream.readLong();
+ RelationalInsertRowNode insertNode = subDeserializeFromWAL(stream);
+ insertNode.setSearchIndex(searchIndex);
+ return insertNode;
+ }
+
+ public static RelationalInsertRowNode deserializeFromWAL(ByteBuffer buffer) {
+ long searchIndex = buffer.getLong();
+ RelationalInsertRowNode insertNode = subDeserializeFromWAL(buffer);
+ insertNode.setSearchIndex(searchIndex);
+ return insertNode;
+ }
+
+ protected static RelationalInsertRowNode subDeserializeFromWAL(DataInputStream stream)
+ throws IOException {
+ // we do not store plan node id in wal entry
+ RelationalInsertRowNode insertNode = new RelationalInsertRowNode(new PlanNodeId(""));
+ insertNode.setTime(stream.readLong());
+ try {
+ insertNode.setDevicePath(
+ DataNodeDevicePathCache.getInstance()
+ .getPartialPath(ReadWriteIOUtils.readString(stream)));
+ } catch (IllegalPathException e) {
+ throw new IllegalArgumentException(DESERIALIZE_ERROR, e);
+ }
+ insertNode.deserializeMeasurementsAndValuesFromWAL(stream);
+ TsTableColumnCategory[] newColumnCategories =
+ new TsTableColumnCategory[insertNode.dataTypes.length];
+ for (int i = 0; i < insertNode.dataTypes.length; i++) {
+ newColumnCategories[i] = TsTableColumnCategory.deserialize(stream);
+ }
+ insertNode.setColumnCategories(newColumnCategories);
+ return insertNode;
+ }
+
+ protected static RelationalInsertRowNode subDeserializeFromWAL(ByteBuffer buffer) {
+ // we do not store plan node id in wal entry
+ RelationalInsertRowNode insertNode = new RelationalInsertRowNode(new PlanNodeId(""));
+ insertNode.setTime(buffer.getLong());
+ try {
+ insertNode.setDevicePath(
+ DataNodeDevicePathCache.getInstance()
+ .getPartialPath(ReadWriteIOUtils.readString(buffer)));
+ } catch (IllegalPathException e) {
+ throw new IllegalArgumentException(DESERIALIZE_ERROR, e);
+ }
+ insertNode.deserializeMeasurementsAndValuesFromWAL(buffer);
+ TsTableColumnCategory[] newColumnCategories =
+ new TsTableColumnCategory[insertNode.dataTypes.length];
+ for (int i = 0; i < insertNode.dataTypes.length; i++) {
+ newColumnCategories[i] = TsTableColumnCategory.deserialize(buffer);
+ }
+ insertNode.setColumnCategories(newColumnCategories);
+ return insertNode;
+ }
+
+ @Override
+ void subSerialize(ByteBuffer buffer) {
+ super.subSerialize(buffer);
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i].serialize(buffer);
+ }
+ }
+
+ @Override
+ void subSerialize(DataOutputStream stream) throws IOException {
+ super.subSerialize(stream);
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i].serialize(stream);
+ }
+ }
+
+ @Override
+ protected void subSerialize(IWALByteBufferView buffer) {
+ super.subSerialize(buffer);
+ for (int i = 0; i < dataTypes.length; i++) {
+ buffer.put(columnCategories[i].getCategory());
+ }
+ }
+
+ public void subDeserialize(ByteBuffer buffer) {
+ super.subDeserialize(buffer);
+ TsTableColumnCategory[] newColumnCategories = new TsTableColumnCategory[dataTypes.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ newColumnCategories[i] = TsTableColumnCategory.deserialize(buffer);
+ }
+ setColumnCategories(newColumnCategories);
+ }
+
+ @Override
+ protected int subSerializeSize() {
+ return super.subSerializeSize() + columnCategories.length * Byte.BYTES;
+ }
+
+ @Override
+ public PlanNodeType getType() {
+ return PlanNodeType.RELATIONAL_INSERT_ROW;
+ }
+
+ public String getTableName() {
+ return devicePath.getFullPath();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
new file mode 100644
index 00000000000..cbbecf511af
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RelationalInsertRowsNode extends InsertRowsNode {
+ // deviceId cache for Table-view insertion
+ private IDeviceID[] deviceIDs;
+
+ public RelationalInsertRowsNode(PlanNodeId id) {
+ super(id);
+ }
+
+ public RelationalInsertRowsNode(
+ PlanNodeId id, List<Integer> insertRowNodeIndexList, List<InsertRowNode> insertRowNodeList) {
+ super(id);
+ this.setInsertRowNodeIndexList(insertRowNodeIndexList);
+ this.setInsertRowNodeList(insertRowNodeList);
+ }
+
+ public IDeviceID getDeviceID(int rowIdx) {
+ if (deviceIDs == null) {
+ deviceIDs = new IDeviceID[getInsertRowNodeList().size()];
+ }
+ if (deviceIDs[rowIdx] == null) {
+ String[] deviceIdSegments = new String[idColumnIndices.size() + 1];
+ deviceIdSegments[0] = this.getTableName();
+ for (int i = 0; i < idColumnIndices.size(); i++) {
+ final Integer columnIndex = idColumnIndices.get(i);
+ deviceIdSegments[i + 1] =
+ ((Object[]) getInsertRowNodeList().get(i).getValues()[columnIndex])[rowIdx].toString();
+ }
+ deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+
+ return deviceIDs[rowIdx];
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitRelationalInsertRows(this, context);
+ }
+
+ public static RelationalInsertRowsNode deserialize(ByteBuffer byteBuffer) {
+ PlanNodeId planNodeId;
+ List<InsertRowNode> insertRowNodeList = new ArrayList<>();
+ List<Integer> insertRowNodeIndex = new ArrayList<>();
+
+ int size = byteBuffer.getInt();
+ for (int i = 0; i < size; i++) {
+ RelationalInsertRowNode insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""));
+ insertRowNode.subDeserialize(byteBuffer);
+ insertRowNodeList.add(insertRowNode);
+ }
+ for (int i = 0; i < size; i++) {
+ insertRowNodeIndex.add(byteBuffer.getInt());
+ }
+
+ planNodeId = PlanNodeId.deserialize(byteBuffer);
+ for (InsertRowNode insertRowNode : insertRowNodeList) {
+ insertRowNode.setPlanNodeId(planNodeId);
+ }
+
+ RelationalInsertRowsNode insertRowsNode = new RelationalInsertRowsNode(planNodeId);
+ insertRowsNode.setInsertRowNodeList(insertRowNodeList);
+ insertRowsNode.setInsertRowNodeIndexList(insertRowNodeIndex);
+ return insertRowsNode;
+ }
+
+ /**
+ * Deserialize from wal.
+ *
+ * @param stream - DataInputStream
+ * @return InsertRowNode
+ * @throws IOException - If an I/O error occurs.
+ * @throws IllegalArgumentException - If meets illegal argument.
+ */
+ public static RelationalInsertRowsNode deserializeFromWAL(DataInputStream stream)
+ throws IOException {
+ // we do not store plan node id in wal entry
+ RelationalInsertRowsNode insertRowsNode = new RelationalInsertRowsNode(new PlanNodeId(""));
+ long searchIndex = stream.readLong();
+ int listSize = stream.readInt();
+ for (int i = 0; i < listSize; i++) {
+ RelationalInsertRowNode insertRowNode = RelationalInsertRowNode.subDeserializeFromWAL(stream);
+ insertRowsNode.addOneInsertRowNode(insertRowNode, i);
+ }
+ insertRowsNode.setSearchIndex(searchIndex);
+ return insertRowsNode;
+ }
+
+ /**
+ * Deserialize from wal.
+ *
+ * @param buffer - ByteBuffer
+ * @return InsertRowNode
+ * @throws IllegalArgumentException - If meets illegal argument
+ */
+ public static RelationalInsertRowsNode deserializeFromWAL(ByteBuffer buffer) {
+ // we do not store plan node id in wal entry
+ RelationalInsertRowsNode insertRowsNode = new RelationalInsertRowsNode(new PlanNodeId(""));
+ long searchIndex = buffer.getLong();
+ int listSize = buffer.getInt();
+ for (int i = 0; i < listSize; i++) {
+ RelationalInsertRowNode insertRowNode = RelationalInsertRowNode.subDeserializeFromWAL(buffer);
+ insertRowsNode.addOneInsertRowNode(insertRowNode, i);
+ }
+ insertRowsNode.setSearchIndex(searchIndex);
+ return insertRowsNode;
+ }
+
+ @Override
+ public PlanNodeType getType() {
+ return PlanNodeType.RELATIONAL_INSERT_ROWS;
+ }
+
+ public String getTableName() {
+ return devicePath.getFullPath();
+ }
+
+ @Override
+ public List<WritePlanNode> splitByPartition(IAnalysis analysis) {
+ Map<TRegionReplicaSet, RelationalInsertRowsNode> splitMap = new HashMap<>();
+ List<TEndPoint> redirectInfo = new ArrayList<>();
+ for (int i = 0; i < getInsertRowNodeList().size(); i++) {
+ InsertRowNode insertRowNode = getInsertRowNodeList().get(i);
+ // Data region for insert row node
+ // each row may belong to different database, pass null for auto-detection
+ TRegionReplicaSet dataRegionReplicaSet =
+ analysis
+ .getDataPartitionInfo()
+ .getDataRegionReplicaSetForWriting(
+ insertRowNode.getDeviceID(),
+ TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()),
+ analysis.getDatabaseName());
+ // Collect redirectInfo
+ redirectInfo.add(dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint());
+ RelationalInsertRowsNode tmpNode = splitMap.get(dataRegionReplicaSet);
+ if (tmpNode != null) {
+ tmpNode.addOneInsertRowNode(insertRowNode, i);
+ } else {
+ tmpNode = new RelationalInsertRowsNode(this.getPlanNodeId());
+ tmpNode.setDataRegionReplicaSet(dataRegionReplicaSet);
+ tmpNode.addOneInsertRowNode(insertRowNode, i);
+ splitMap.put(dataRegionReplicaSet, tmpNode);
+ }
+ }
+ analysis.setRedirectNodeList(redirectInfo);
+
+ return new ArrayList<>(splitMap.values());
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
new file mode 100644
index 00000000000..6d0b6cccbb4
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.exception.query.OutOfTTLException;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.IntToLongFunction;
+
+public class RelationalInsertTabletNode extends InsertTabletNode {
+
+ // deviceId cache for Table-view insertion
+ private IDeviceID[] deviceIDs;
+
+ public RelationalInsertTabletNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ long[] times,
+ BitMap[] bitMaps,
+ Object[] columns,
+ int rowCount,
+ TsTableColumnCategory[] columnCategories) {
+ super(id, devicePath, isAligned, measurements, dataTypes, times, bitMaps, columns, rowCount);
+ setColumnCategories(columnCategories);
+ }
+
+ public RelationalInsertTabletNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ MeasurementSchema[] measurementSchemas,
+ long[] times,
+ BitMap[] bitMaps,
+ Object[] columns,
+ int rowCount,
+ TsTableColumnCategory[] columnCategories) {
+ super(
+ id,
+ devicePath,
+ isAligned,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ times,
+ bitMaps,
+ columns,
+ rowCount);
+ setColumnCategories(columnCategories);
+ }
+
+ public RelationalInsertTabletNode(PlanNodeId id) {
+ super(id);
+ }
+
+ public IDeviceID getDeviceID(int rowIdx) {
+ if (deviceIDs == null) {
+ deviceIDs = new IDeviceID[rowCount];
+ }
+ if (deviceIDs[rowIdx] == null) {
+ String[] deviceIdSegments = new String[idColumnIndices.size() + 1];
+ deviceIdSegments[0] = this.getTableName();
+ for (int i = 0; i < idColumnIndices.size(); i++) {
+ final Integer columnIndex = idColumnIndices.get(i);
+ Object idSeg = ((Object[]) columns[columnIndex])[rowIdx];
+ deviceIdSegments[i + 1] = idSeg != null ? idSeg.toString() : null;
+ }
+ deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+
+ return deviceIDs[rowIdx];
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitRelationalInsertTablet(this, context);
+ }
+
+ protected InsertTabletNode getEmptySplit(int count) {
+ long[] subTimes = new long[count];
+ Object[] values = initTabletValues(dataTypes.length, count, dataTypes);
+ BitMap[] newBitMaps = this.bitMaps == null ? null : initBitmaps(dataTypes.length, count);
+ return new RelationalInsertTabletNode(
+ getPlanNodeId(),
+ devicePath,
+ isAligned,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ subTimes,
+ newBitMaps,
+ values,
+ subTimes.length,
+ columnCategories);
+ }
+
+ public static RelationalInsertTabletNode deserialize(ByteBuffer byteBuffer) {
+ RelationalInsertTabletNode insertNode = new RelationalInsertTabletNode(new PlanNodeId(""));
+ insertNode.subDeserialize(byteBuffer);
+ insertNode.setPlanNodeId(PlanNodeId.deserialize(byteBuffer));
+ return insertNode;
+ }
+
+ @Override
+ protected void serializeAttributes(ByteBuffer byteBuffer) {
+ super.serializeAttributes(byteBuffer);
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i].serialize(byteBuffer);
+ }
+ }
+
+ @Override
+ protected void serializeAttributes(DataOutputStream stream) throws IOException {
+ super.serializeAttributes(stream);
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i].serialize(stream);
+ }
+ }
+
+ public void subDeserialize(ByteBuffer buffer) {
+ super.subDeserialize(buffer);
+ TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[dataTypes.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i] = TsTableColumnCategory.deserialize(buffer);
+ }
+ setColumnCategories(columnCategories);
+ }
+
+ void subSerialize(IWALByteBufferView buffer, int start, int end) {
+ super.subSerialize(buffer, start, end);
+ for (int i = 0; i < dataTypes.length; i++) {
+ buffer.put(columnCategories[i].getCategory());
+ }
+ }
+
+ @Override
+ protected void subDeserializeFromWAL(ByteBuffer buffer) {
+ super.subDeserializeFromWAL(buffer);
+ TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[dataTypes.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i] = TsTableColumnCategory.deserialize(buffer);
+ }
+ setColumnCategories(columnCategories);
+ }
+
+ @Override
+ protected void subDeserializeFromWAL(DataInputStream stream) throws IOException {
+ super.subDeserializeFromWAL(stream);
+ TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[dataTypes.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ columnCategories[i] = TsTableColumnCategory.deserialize(stream);
+ }
+ setColumnCategories(columnCategories);
+ }
+
+ /** Deserialize from wal */
+ public static RelationalInsertTabletNode deserializeFromWAL(DataInputStream stream)
+ throws IOException {
+ // we do not store plan node id in wal entry
+ RelationalInsertTabletNode insertNode = new RelationalInsertTabletNode(new PlanNodeId(""));
+ insertNode.subDeserializeFromWAL(stream);
+ return insertNode;
+ }
+
+ public static RelationalInsertTabletNode deserializeFromWAL(ByteBuffer buffer) {
+ // we do not store plan node id in wal entry
+ RelationalInsertTabletNode insertNode = new RelationalInsertTabletNode(new PlanNodeId(""));
+ insertNode.subDeserializeFromWAL(buffer);
+ return insertNode;
+ }
+
+ @Override
+ int subSerializeSize(int start, int end) {
+ return super.subSerializeSize(start, end) + columnCategories.length * Byte.BYTES;
+ }
+
+ @Override
+ public PlanNodeType getType() {
+ return PlanNodeType.RELATIONAL_INSERT_TABLET;
+ }
+
+ public List<Pair<IDeviceID, Integer>> splitByDevice(int start, int end) {
+ List<Pair<IDeviceID, Integer>> result = new ArrayList<>();
+ IDeviceID prevDeviceId = getDeviceID(start);
+
+ int i = start + 1;
+ for (; i < end; i++) {
+ IDeviceID currentDeviceId = getDeviceID(i);
+ if (!currentDeviceId.equals(prevDeviceId)) {
+ result.add(new Pair<>(prevDeviceId, i));
+ prevDeviceId = getDeviceID(i);
+ }
+ }
+ result.add(new Pair<>(prevDeviceId, end));
+
+ return result;
+ }
+
+ @Override
+ public int checkTTL(TSStatus[] results, IntToLongFunction rowTTLGetter) throws OutOfTTLException {
+ return checkTTLInternal(results, rowTTLGetter, false);
+ }
+
+ public String getTableName() {
+ return devicePath.getFullPath();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
index 095248a7391..5c5f51e776b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
@@ -94,6 +94,9 @@ import static java.util.Objects.requireNonNull;
public class Analysis implements IAnalysis {
+ private String databaseName;
+ private List<TEndPoint> redirectNodeList;
+
@Nullable private Statement root;
private final Map<NodeRef<Parameter>, Expression> parameters;
@@ -172,12 +175,18 @@ public class Analysis implements IAnalysis {
// indicate if value filter exists in query
private boolean hasValueFilter = false;
+ private TSStatus failStatus;
+
// indicate if sort node exists in query
private boolean hasSortNode = false;
// if emptyDataSource, there is no need to execute the query in BE
private boolean emptyDataSource = false;
+ public DataPartition getDataPartition() {
+ return dataPartition;
+ }
+
public Analysis(@Nullable Statement root, Map<NodeRef<Parameter>, Expression> parameters) {
this.root = root;
this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null"));
@@ -610,12 +619,17 @@ public class Analysis implements IAnalysis {
@Override
public boolean isFailed() {
- return false;
+ return failStatus != null;
}
@Override
public TSStatus getFailStatus() {
- return null;
+ return failStatus;
+ }
+
+ @Override
+ public void setFailStatus(TSStatus failStatus) {
+ this.failStatus = failStatus;
}
@Override
@@ -627,10 +641,20 @@ public class Analysis implements IAnalysis {
this.finishQueryAfterAnalyze = true;
}
+ @Override
+ public void setFinishQueryAfterAnalyze(boolean finishQueryAfterAnalyze) {
+ this.finishQueryAfterAnalyze = finishQueryAfterAnalyze;
+ }
+
public boolean isFinishQueryAfterAnalyze() {
return finishQueryAfterAnalyze;
}
+ @Override
+ public void setDataPartitionInfo(DataPartition dataPartition) {
+ this.dataPartition = dataPartition;
+ }
+
@Override
public TsBlock constructResultForMemorySource(MPPQueryContext context) {
requireNonNull(getStatement(), "root statement is analysis is null");
@@ -694,12 +718,15 @@ public class Analysis implements IAnalysis {
@Override
public void setRedirectNodeList(List<TEndPoint> redirectNodeList) {
- throw new UnsupportedOperationException();
+ this.redirectNodeList = redirectNodeList;
}
@Override
public void addEndPointToRedirectNodeList(TEndPoint endPoint) {
- throw new UnsupportedOperationException();
+ if (redirectNodeList == null) {
+ redirectNodeList = new ArrayList<>();
+ }
+ redirectNodeList.add(endPoint);
}
@Override
@@ -957,4 +984,14 @@ public class Analysis implements IAnalysis {
return unmodifiableList(quantifiedComparisonSubqueries);
}
}
+
+ @Override
+ public void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+
+ @Override
+ public String getDatabaseName() {
+ return databaseName;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analyzer.java
index 91f625a7fe9..88f235902e0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analyzer.java
@@ -57,6 +57,7 @@ public class Analyzer {
public Analysis analyze(Statement statement) {
Analysis analysis = new Analysis(statement, parameterLookup);
+ analysis.setDatabaseName(session.getDatabaseName().get());
StatementAnalyzer analyzer =
statementAnalyzerFactory.createStatementAnalyzer(
analysis, session, warningCollector, CorrelationSupport.ALLOWED);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java
index ac712bbf8c1..cf1350f51a5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java
@@ -1331,6 +1331,7 @@ public class ExpressionAnalyzer {
Map<NodeRef<Parameter>, Expression> parameters,
WarningCollector warningCollector) {
Analysis analysis = new Analysis(null, parameters);
+ analysis.setDatabaseName(session.getDatabaseName().get());
ExpressionAnalyzer analyzer =
new ExpressionAnalyzer(
metadata,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index badb1e1a312..57d10e46fd0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -21,9 +21,12 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.execution.warnings.IoTDBWarning;
import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector;
+import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils;
+import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName;
@@ -59,6 +62,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingElement;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria;
@@ -100,6 +106,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
@@ -156,7 +164,7 @@ public class StatementAnalyzer {
private final StatementAnalyzerFactory statementAnalyzerFactory;
- private final Analysis analysis;
+ private Analysis analysis;
private final AccessControl accessControl;
@@ -215,6 +223,7 @@ public class StatementAnalyzer {
* scopes hierarchy should always have outer query scope (if provided) as ancestor.
*/
private final class Visitor extends AstVisitor<Scope, Optional<Scope>> {
+
private final boolean isTopLevel;
private final Optional<Scope> outerQueryScope;
private final WarningCollector warningCollector;
@@ -360,6 +369,41 @@ public class StatementAnalyzer {
throw new SemanticException("Insert statement is not supported yet.");
}
+ @Override
+ protected Scope visitInsertRow(InsertRow node, Optional<Scope> context) {
+ return visitInsert(node, context);
+ }
+
+ protected Scope visitInsertTablet(InsertTablet insert, Optional<Scope> scope) {
+ return visitInsert(insert, scope);
+ }
+
+ @Override
+ protected Scope visitInsertRows(InsertRows node, Optional<Scope> context) {
+ return visitInsert(node, context);
+ }
+
+ private Scope visitInsert(WrappedInsertStatement insert, Optional<Scope> scope) {
+ final Scope ret = Scope.create();
+
+ final MPPQueryContext context = insert.getContext();
+ InsertBaseStatement innerInsert = insert.getInnerTreeStatement();
+
+ innerInsert =
+ AnalyzeUtils.analyzeInsert(
+ context,
+ innerInsert,
+ () -> SchemaValidator.validate(metadata, insert, context),
+ metadata::getOrCreateDataPartition,
+ AnalyzeUtils::computeTableDataPartitionParams,
+ analysis,
+ false);
+ insert.setInnerTreeStatement(innerInsert);
+ analysis.setScope(insert, ret);
+
+ return ret;
+ }
+
@Override
protected Scope visitDelete(Delete node, Optional<Scope> scope) {
throw new SemanticException("Delete statement is not supported yet.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzerFactory.java
index b777c91584b..f37b6460994 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzerFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzerFactory.java
@@ -36,7 +36,7 @@ public class StatementAnalyzerFactory {
public StatementAnalyzerFactory(
Metadata metadata, SqlParser sqlParser, AccessControl accessControl) {
this.metadata = requireNonNull(metadata, "plannerContext is null");
- this.sqlParser = requireNonNull(sqlParser, "sqlParser is null");
+ this.sqlParser = sqlParser;
this.accessControl = requireNonNull(accessControl, "accessControl is null");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java
index 715c9488127..ed224017b12 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.relational.metadata;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.tsfile.read.common.type.BinaryType;
import org.apache.tsfile.read.common.type.BlobType;
@@ -31,6 +32,7 @@ import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TimestampType;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeEnum;
+import org.apache.tsfile.read.common.type.TypeFactory;
import org.apache.tsfile.read.common.type.UnknownType;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -54,7 +56,6 @@ public class ColumnSchema {
public ColumnSchema(
String name, Type type, boolean hidden, TsTableColumnCategory columnCategory) {
requireNonNull(name, "name is null");
- requireNonNull(type, "type is null");
this.name = name.toLowerCase(ENGLISH);
this.type = type;
@@ -156,6 +157,14 @@ public class ColumnSchema {
}
}
+ public static ColumnSchema ofTsColumnSchema(TsTableColumnSchema schema) {
+ return new ColumnSchema(
+ schema.getColumnName(),
+ TypeFactory.getType(schema.getDataType()),
+ false,
+ schema.getColumnCategory());
+ }
+
public static Builder builder() {
return new Builder();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
index 62eee0d46a2..d5fb5df4de6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
@@ -117,6 +117,18 @@ public interface Metadata {
*/
void validateDeviceSchema(ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context);
+ /**
+ * Get or create data partition, used in cluster write scenarios. if enableAutoCreateSchema is
+ * true and database/series/time slots not exists, then automatically create.
+ *
+ * @param dataPartitionQueryParams the list of DataPartitionQueryParams
+ * @param userName
+ */
+ default DataPartition getOrCreateDataPartition(
+ List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) {
+ throw new UnsupportedOperationException();
+ }
+
// ======================== Table Model Schema Partition Interface ========================
/**
* Get or create schema partition, used in data insertion with enable_auto_create_schema is true.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index 8ed28f512a4..12a9bd73bab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -327,6 +327,12 @@ public class TableMetadataImpl implements Metadata {
TableDeviceSchemaValidator.getInstance().validateDeviceSchema(schemaValidation, context);
}
+ @Override
+ public DataPartition getOrCreateDataPartition(
+ List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) {
+ return partitionFetcher.getOrCreateDataPartition(dataPartitionQueryParams, userName);
+ }
+
@Override
public SchemaPartition getOrCreateSchemaPartition(
String database, List<IDeviceID> deviceIDList, String userName) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
index e3ed509931c..e2ff268ef55 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
@@ -19,7 +19,19 @@
package org.apache.iotdb.db.queryengine.plan.relational.metadata;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
+
+import org.apache.tsfile.write.record.Tablet.ColumnType;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
public class TableSchema {
@@ -39,4 +51,59 @@ public class TableSchema {
public List<ColumnSchema> getColumns() {
return columns;
}
+
+ public static TableSchema of(TsTable tsTable) {
+ String tableName = tsTable.getTableName();
+ List<ColumnSchema> columns = new ArrayList<>();
+ for (TsTableColumnSchema tsTableColumnSchema : tsTable.getColumnList()) {
+ columns.add(ColumnSchema.ofTsColumnSchema(tsTableColumnSchema));
+ }
+ return new TableSchema(tableName, columns);
+ }
+
+ public org.apache.tsfile.file.metadata.TableSchema toTsFileTableSchema() {
+ // TODO-Table: unify redundant definitions
+ String tableName = this.getTableName();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<ColumnType> columnTypes = new ArrayList<>();
+ for (ColumnSchema column : columns) {
+ if (column.getColumnCategory() == TsTableColumnCategory.TIME) {
+ continue;
+ }
+ measurementSchemas.add(
+ new MeasurementSchema(
+ column.getName(), InternalTypeManager.getTSDataType(column.getType())));
+ columnTypes.add(column.getColumnCategory().toTsFileColumnType());
+ }
+ return new org.apache.tsfile.file.metadata.TableSchema(
+ tableName, measurementSchemas, columnTypes);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableSchema that = (TableSchema) o;
+ return Objects.equals(tableName, that.tableName) && Objects.equals(columns, that.columns);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableName, columns);
+ }
+
+ @Override
+ public String toString() {
+ return "TableSchema{" + "tableName='" + tableName + '\'' + ", columns=" + columns + '}';
+ }
+
+ public List<ColumnSchema> getIdColumns() {
+ return columns.stream()
+ .filter(c -> c.getColumnCategory() == TsTableColumnCategory.ID)
+ .collect(Collectors.toList());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
index a941dc2a272..b6faadf5ebb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.table.column.IdColumnSchema;
import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager;
@@ -66,6 +67,7 @@ public class TableHeaderSchemaValidator {
}
private static class TableHeaderSchemaValidatorHolder {
+
private static final TableHeaderSchemaValidator INSTANCE = new TableHeaderSchemaValidator();
}
@@ -87,7 +89,6 @@ public class TableHeaderSchemaValidator {
}
TsTable table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName());
List<ColumnSchema> missingColumnList = new ArrayList<>();
- List<ColumnSchema> resultColumnList = new ArrayList<>();
// first round validate, check existing schema
if (table == null) {
@@ -127,10 +128,13 @@ public class TableHeaderSchemaValidator {
}
}
- if (!missingColumnList.isEmpty()) {
+ List<ColumnSchema> resultColumnList = new ArrayList<>();
+ if (!missingColumnList.isEmpty()
+ && IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()) {
// TODO table metadata: authority check for table alter
// check id or attribute column data type in this method
autoCreateColumn(database, tableSchema.getTableName(), missingColumnList, context);
+ resultColumnList.addAll(missingColumnList);
}
table
@@ -226,7 +230,10 @@ public class TableHeaderSchemaValidator {
if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
throw new RuntimeException(
new IoTDBException(
- "Auto add table column failed.", result.getStatusCode().getStatusCode()));
+ String.format(
+ "Auto add table column failed: %s.%s, %s",
+ database, tableName, inputColumnList),
+ result.getStatusCode().getStatusCode()));
}
} catch (ExecutionException | InterruptedException e) {
LOGGER.warn("Auto add table column failed.", e);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
index c88dc681a13..a906cd53e31 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceFetchNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceQueryNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
@@ -43,6 +44,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedStatement;
import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
@@ -83,20 +85,22 @@ public class LogicalPlanner {
public LogicalQueryPlan plan(Analysis analysis) {
PlanNode planNode = planStatement(analysis, analysis.getStatement());
- for (PlanOptimizer optimizer : planOptimizers) {
- planNode =
- optimizer.optimize(
- planNode,
- new PlanOptimizer.Context(
- sessionInfo,
- analysis,
- metadata,
- queryContext,
- queryContext.getTypeProvider(),
- symbolAllocator,
- queryContext.getQueryId(),
- warningCollector,
- PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector()));
+ if (analysis.getStatement() instanceof Query) {
+ for (PlanOptimizer optimizer : planOptimizers) {
+ planNode =
+ optimizer.optimize(
+ planNode,
+ new PlanOptimizer.Context(
+ sessionInfo,
+ analysis,
+ metadata,
+ queryContext,
+ queryContext.getTypeProvider(),
+ symbolAllocator,
+ queryContext.getQueryId(),
+ warningCollector,
+ PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector()));
+ }
}
return new LogicalQueryPlan(queryContext, planNode);
@@ -122,11 +126,17 @@ public class LogicalPlanner {
if (statement instanceof Explain) {
return createRelationPlan(analysis, (Query) ((Explain) statement).getStatement());
}
+ if (statement instanceof WrappedStatement) {
+ return createRelationPlan(analysis, ((WrappedStatement) statement));
+ }
throw new IllegalStateException(
"Unsupported statement type: " + statement.getClass().getSimpleName());
}
private PlanNode createOutputPlan(RelationPlan plan, Analysis analysis) {
+ if (plan.getRoot() instanceof WritePlanNode) {
+ return plan.getRoot();
+ }
ImmutableList.Builder<Symbol> outputs = ImmutableList.builder();
ImmutableList.Builder<String> names = ImmutableList.builder();
List<ColumnHeader> columnHeaders = new ArrayList<>();
@@ -160,6 +170,10 @@ public class LogicalPlanner {
return outputNode;
}
+ private RelationPlan createRelationPlan(Analysis analysis, WrappedStatement statement) {
+ return getRelationPlanner(analysis).process(statement, null);
+ }
+
private RelationPlan createRelationPlan(Analysis analysis, Query query) {
return getRelationPlanner(analysis).process(query, null);
}
@@ -230,6 +244,8 @@ public class LogicalPlanner {
List<ColumnHeader> columnHeaderList =
getColumnHeaderList(statement.getDatabase(), statement.getTableName());
+ analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true));
+
TableDeviceQueryNode queryNode =
new TableDeviceQueryNode(
queryContext.getQueryId().genPlanNodeId(),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index 8b974707298..a1c98cb14fc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -17,6 +17,10 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef;
@@ -27,6 +31,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNod
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
@@ -38,11 +45,16 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Union;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -50,6 +62,7 @@ import java.util.Map;
import static java.util.Objects.requireNonNull;
public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
+
private final Analysis analysis;
private final SymbolAllocator symbolAllocator;
private final MPPQueryContext queryContext;
@@ -202,4 +215,64 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
protected RelationPlan visitExcept(Except node, Void context) {
throw new IllegalStateException("Except is not supported in current version.");
}
+
+ @Override
+ protected RelationPlan visitInsertTablet(InsertTablet node, Void context) {
+ final InsertTabletStatement insertTabletStatement = node.getInnerTreeStatement();
+ RelationalInsertTabletNode insertNode =
+ new RelationalInsertTabletNode(
+ idAllocator.genPlanNodeId(),
+ insertTabletStatement.getDevicePath(),
+ insertTabletStatement.isAligned(),
+ insertTabletStatement.getMeasurements(),
+ insertTabletStatement.getDataTypes(),
+ insertTabletStatement.getMeasurementSchemas(),
+ insertTabletStatement.getTimes(),
+ insertTabletStatement.getBitMaps(),
+ insertTabletStatement.getColumns(),
+ insertTabletStatement.getRowCount(),
+ insertTabletStatement.getColumnCategories());
+ insertNode.setFailedMeasurementNumber(insertTabletStatement.getFailedMeasurementNumber());
+ return new RelationPlan(insertNode, analysis.getRootScope(), Collections.emptyList());
+ }
+
+ @Override
+ protected RelationPlan visitInsertRow(InsertRow node, Void context) {
+ InsertRowStatement insertRowStatement = node.getInnerTreeStatement();
+ RelationalInsertRowNode insertNode = fromInsertRowStatement(insertRowStatement);
+ return new RelationPlan(insertNode, analysis.getRootScope(), Collections.emptyList());
+ }
+
+ protected RelationalInsertRowNode fromInsertRowStatement(InsertRowStatement insertRowStatement) {
+ RelationalInsertRowNode insertNode =
+ new RelationalInsertRowNode(
+ idAllocator.genPlanNodeId(),
+ insertRowStatement.getDevicePath(),
+ insertRowStatement.isAligned(),
+ insertRowStatement.getMeasurements(),
+ insertRowStatement.getDataTypes(),
+ insertRowStatement.getTime(),
+ insertRowStatement.getValues(),
+ insertRowStatement.isNeedInferType(),
+ insertRowStatement.getColumnCategories());
+ insertNode.setFailedMeasurementNumber(insertRowStatement.getFailedMeasurementNumber());
+ insertNode.setMeasurementSchemas(insertRowStatement.getMeasurementSchemas());
+ return insertNode;
+ }
+
+ @Override
+ protected RelationPlan visitInsertRows(InsertRows node, Void context) {
+ InsertRowsStatement insertRowsStatement = node.getInnerTreeStatement();
+ List<Integer> indices = new ArrayList<>();
+ List<InsertRowNode> insertRowStatements = new ArrayList<>();
+ for (int i = 0; i < insertRowsStatement.getInsertRowStatementList().size(); i++) {
+ indices.add(i);
+ insertRowStatements.add(
+ fromInsertRowStatement(insertRowsStatement.getInsertRowStatementList().get(i)));
+ }
+ RelationalInsertRowsNode relationalInsertRowsNode =
+ new RelationalInsertRowsNode(idAllocator.genPlanNodeId(), indices, insertRowStatements);
+ return new RelationPlan(
+ relationalInsertRowsNode, analysis.getRootScope(), Collections.emptyList());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/DistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/DistributedPlanGenerator.java
index f37c384e4d4..42ba1362e8b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/DistributedPlanGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/DistributedPlanGenerator.java
@@ -17,6 +17,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.commons.partition.SchemaPartition;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistribution;
@@ -26,7 +27,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceFetchNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceQueryNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceSourceNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry;
import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme;
@@ -60,7 +60,6 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static com.google.common.collect.ImmutableList.toImmutableList;
-import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT;
import static org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator.parseDeviceIdArray;
import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PushPredicateIntoTableScan.containsDiffFunction;
import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR;
@@ -83,6 +82,7 @@ public class DistributedPlanGenerator
return res;
} else if (res.size() > 1) {
CollectNode collectNode = new CollectNode(queryId.genPlanNodeId());
+ collectNode.setOutputSymbols(res.get(0).getOutputSymbols());
res.forEach(collectNode::addChild);
return Collections.singletonList(collectNode);
} else {
@@ -358,6 +358,7 @@ public class DistributedPlanGenerator
// children has no sort property, use CollectNode to merge children
CollectNode collectNode = new CollectNode(queryId.genPlanNodeId());
+ collectNode.setOutputSymbols(firstChild.getOutputSymbols());
childrenNodes.forEach(collectNode::addChild);
return collectNode;
}
@@ -445,8 +446,7 @@ public class DistributedPlanGenerator
// ------------------- schema related interface ---------------------------------------------
@Override
public List<PlanNode> visitTableDeviceQuery(TableDeviceQueryNode node, PlanContext context) {
- String database =
- ROOT + "." + ((TableDeviceSourceNode) node.getChildren().get(0)).getDatabase();
+ String database = PathUtils.qualifyDatabaseName(node.getDatabase());
Set<TRegionReplicaSet> schemaRegionSet = new HashSet<>();
analysis
.getSchemaPartitionInfo()
@@ -473,8 +473,7 @@ public class DistributedPlanGenerator
@Override
public List<PlanNode> visitTableDeviceFetch(TableDeviceFetchNode node, PlanContext context) {
- String database =
- ROOT + "." + ((TableDeviceSourceNode) node.getChildren().get(0)).getDatabase();
+ String database = PathUtils.qualifyDatabaseName(node.getDatabase());
Set<TRegionReplicaSet> schemaRegionSet = new HashSet<>();
SchemaPartition schemaPartition = analysis.getSchemaPartitionInfo();
Map<TSeriesPartitionSlot, TRegionReplicaSet> databaseMap =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java
index 04caab49710..5b23aac3e3c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan;
import org.apache.iotdb.db.queryengine.plan.planner.plan.SubPlan;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
@@ -43,6 +44,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector.NOOP;
public class TableDistributionPlanner {
+
private final Analysis analysis;
private final LogicalQueryPlan logicalQueryPlan;
private final MPPQueryContext mppQueryContext;
@@ -103,7 +105,9 @@ public class TableDistributionPlanner {
List<FragmentInstance> fragmentInstances =
mppQueryContext.getQueryType() == QueryType.READ
? new TableModelQueryFragmentPlanner(subPlan, analysis, mppQueryContext).plan()
- : new WriteFragmentParallelPlanner(subPlan, analysis, mppQueryContext).parallelPlan();
+ : new WriteFragmentParallelPlanner(
+ subPlan, analysis, mppQueryContext, WritePlanNode::splitByPartition)
+ .parallelPlan();
// only execute this step for READ operation
if (mppQueryContext.getQueryType() == QueryType.READ) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CollectNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CollectNode.java
index bd7cb33d78f..668ee384909 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CollectNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CollectNode.java
@@ -27,10 +27,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChild
import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
import com.google.common.base.Objects;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
import static com.google.common.base.Preconditions.checkArgument;
@@ -41,6 +43,8 @@ import static com.google.common.base.Preconditions.checkArgument;
*/
public class CollectNode extends MultiChildProcessNode {
+ private List<Symbol> outputSymbols;
+
public CollectNode(PlanNodeId id) {
super(id);
}
@@ -56,12 +60,18 @@ public class CollectNode extends MultiChildProcessNode {
@Override
public PlanNode clone() {
- return new CollectNode(id);
+ CollectNode collectNode = new CollectNode(id);
+ collectNode.outputSymbols = outputSymbols;
+ return collectNode;
+ }
+
+ public void setOutputSymbols(List<Symbol> outputSymbols) {
+ this.outputSymbols = outputSymbols;
}
@Override
public List<Symbol> getOutputSymbols() {
- return children.get(0).getOutputSymbols();
+ return outputSymbols;
}
@Override
@@ -78,16 +88,29 @@ public class CollectNode extends MultiChildProcessNode {
@Override
protected void serializeAttributes(ByteBuffer byteBuffer) {
PlanNodeType.TABLE_COLLECT_NODE.serialize(byteBuffer);
+ ReadWriteIOUtils.write(outputSymbols.size(), byteBuffer);
+ outputSymbols.forEach(symbol -> Symbol.serialize(symbol, byteBuffer));
}
@Override
protected void serializeAttributes(DataOutputStream stream) throws IOException {
PlanNodeType.TABLE_COLLECT_NODE.serialize(stream);
+ ReadWriteIOUtils.write(outputSymbols.size(), stream);
+ for (Symbol symbol : outputSymbols) {
+ Symbol.serialize(symbol, stream);
+ }
}
public static CollectNode deserialize(ByteBuffer byteBuffer) {
PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
- return new CollectNode(planNodeId);
+ int size = ReadWriteIOUtils.readInt(byteBuffer);
+ List<Symbol> outputs = new ArrayList<>(size);
+ while (size-- > 0) {
+ outputs.add(Symbol.deserialize(byteBuffer));
+ }
+ CollectNode collectNode = new CollectNode(planNodeId);
+ collectNode.setOutputSymbols(outputs);
+ return collectNode;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
index 6de1ef903ce..a01e6b1c5df 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
@@ -153,10 +153,9 @@ public class CreateTableDeviceNode extends WritePlanNode {
if (partitionKeyList == null) {
List<IDeviceID> tmpPartitionKeyList = new ArrayList<>();
for (Object[] rawId : deviceIdList) {
- String[] partitionKey = new String[rawId.length + 1];
- partitionKey[0] = tableName;
+ String[] partitionKey = new String[rawId.length];
for (int i = 0; i < rawId.length; i++) {
- partitionKey[i + 1] = (String) rawId[i];
+ partitionKey[i] = (String) rawId[i];
}
tmpPartitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
index 74ff26c7dce..6dfec749d5d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicateCombineIntoTableScanChecker;
@@ -244,6 +246,16 @@ public class PushPredicateIntoTableScan implements PlanOptimizer {
return node;
}
+ @Override
+ public PlanNode visitInsertTablet(InsertTabletNode node, Void context) {
+ return node;
+ }
+
+ @Override
+ public PlanNode visitRelationalInsertTablet(RelationalInsertTabletNode node, Void context) {
+ return node;
+ }
+
/** Get deviceEntries and DataPartition used in TableScan. */
private void tableMetadataIndexScan(TableScanNode node, List<Expression> metadataExpressions) {
List<String> attributeColumns =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
index dcf2c6c36d5..7989c23e85d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
@@ -387,6 +387,22 @@ public abstract class AstVisitor<R, C> {
return visitStatement(node, context);
}
+ protected R visitInsertTablet(InsertTablet node, C context) {
+ return visitStatement(node, context);
+ }
+
+ protected R visitFlush(Flush node, C context) {
+ return visitStatement(node, context);
+ }
+
+ protected R visitInsertRow(InsertRow node, C context) {
+ return visitStatement(node, context);
+ }
+
+ protected R visitInsertRows(InsertRows node, C context) {
+ return visitStatement(node, context);
+ }
+
protected R visitDelete(Delete node, C context) {
return visitStatement(node, context);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
index bb27ea604b4..22aa5c8107b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingExcepti
import com.google.common.base.CharMatcher;
import com.google.common.io.BaseEncoding;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
@@ -130,4 +131,9 @@ public class BinaryLiteral extends Literal {
value[i] = ReadWriteIOUtils.readByte(byteBuffer);
}
}
+
+ @Override
+ public Object getTsValue() {
+ return new Binary(value);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BooleanLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BooleanLiteral.java
index 9ac55f61aca..75ad655743c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BooleanLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BooleanLiteral.java
@@ -104,4 +104,9 @@ public class BooleanLiteral extends Literal {
super(null);
this.value = ReadWriteIOUtils.readBool(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return value;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DecimalLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DecimalLiteral.java
index 2841f81221b..f7f99e0a1a0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DecimalLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DecimalLiteral.java
@@ -19,11 +19,13 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
import static java.util.Objects.requireNonNull;
@@ -93,4 +95,9 @@ public class DecimalLiteral extends Literal {
super(null);
this.value = ReadWriteIOUtils.readString(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return new Binary(value.getBytes(StandardCharsets.UTF_8));
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DoubleLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DoubleLiteral.java
index d744ad23c8d..d7916148a6c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DoubleLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DoubleLiteral.java
@@ -98,4 +98,9 @@ public class DoubleLiteral extends Literal {
super(null);
this.value = ReadWriteIOUtils.readDouble(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return value;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FetchDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FetchDevice.java
index d7c37ac1d14..f416ac8e9ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FetchDevice.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FetchDevice.java
@@ -58,10 +58,9 @@ public class FetchDevice extends Statement {
if (partitionKeyList == null) {
List<IDeviceID> partitionKeyList = new ArrayList<>();
for (Object[] rawId : deviceIdList) {
- String[] partitionKey = new String[rawId.length + 1];
- partitionKey[0] = tableName;
+ String[] partitionKey = new String[rawId.length];
for (int i = 0; i < rawId.length; i++) {
- partitionKey[i + 1] = Objects.toString(rawId[i].toString());
+ partitionKey[i] = Objects.toString(rawId[i]);
}
partitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Flush.java
similarity index 72%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Flush.java
index 71d130ad228..608d4e34ce8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Flush.java
@@ -19,24 +19,17 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
-import com.google.common.collect.ImmutableList;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import javax.annotation.Nullable;
+public class Flush extends WrappedStatement {
-import java.util.List;
-
-public abstract class Literal extends Expression {
- protected Literal(@Nullable NodeLocation location) {
- super(location);
+ public Flush(Statement innerTreeStatement, MPPQueryContext context) {
+ super(innerTreeStatement, context);
}
@Override
public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
- return visitor.visitLiteral(this, context);
- }
-
- @Override
- public List<Node> getChildren() {
- return ImmutableList.of();
+ return visitor.visitFlush(this, context);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/GenericLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/GenericLiteral.java
index cd251616558..25e41093c8b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/GenericLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/GenericLiteral.java
@@ -21,11 +21,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
import static java.util.Objects.requireNonNull;
@@ -122,4 +124,9 @@ public class GenericLiteral extends Literal {
this.type = ReadWriteIOUtils.readString(byteBuffer);
this.value = ReadWriteIOUtils.readString(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return new Binary(value.getBytes(StandardCharsets.UTF_8));
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java
new file mode 100644
index 00000000000..9a8a7367b6d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class InsertRow extends WrappedInsertStatement {
+
+ public InsertRow(InsertRowStatement insertRowStatement, MPPQueryContext context) {
+ super(insertRowStatement, context);
+ }
+
+ @Override
+ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
+ return visitor.visitInsertRow(this, context);
+ }
+
+ @Override
+ public InsertRowStatement getInnerTreeStatement() {
+ return ((InsertRowStatement) super.getInnerTreeStatement());
+ }
+
+ @Override
+ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryProcessException {
+ getInnerTreeStatement().updateAfterSchemaValidation(context);
+ }
+
+ @Override
+ public String getDatabase() {
+ return context.getSession().getDatabaseName().orElse(null);
+ }
+
+ @Override
+ public String getTableName() {
+ return getInnerTreeStatement().getDevicePath().getFullPath();
+ }
+
+ @Override
+ public List<Object[]> getDeviceIdList() {
+ final InsertRowStatement insertRowStatement = getInnerTreeStatement();
+ return Collections.singletonList(insertRowStatement.getTableDeviceID().getSegments());
+ }
+
+ @Override
+ public List<String> getAttributeColumnNameList() {
+ final InsertRowStatement insertRowStatement = getInnerTreeStatement();
+ List<String> result = new ArrayList<>();
+ for (int i = 0; i < insertRowStatement.getColumnCategories().length; i++) {
+ if (insertRowStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) {
+ result.add(insertRowStatement.getMeasurements()[i]);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<Object[]> getAttributeValueList() {
+ final InsertRowStatement insertRowStatement = getInnerTreeStatement();
+ final List<Integer> attrColumnIndices = insertRowStatement.getAttrColumnIndices();
+ Object[] attrValues = new Object[attrColumnIndices.size()];
+ for (int j = 0; j < attrColumnIndices.size(); j++) {
+ final int columnIndex = attrColumnIndices.get(j);
+ attrValues[j] = insertRowStatement.getValues()[columnIndex];
+ }
+ return Collections.singletonList(attrValues);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java
new file mode 100644
index 00000000000..184181f5de6
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class InsertRows extends WrappedInsertStatement {
+
+ public InsertRows(InsertRowsStatement insertRowsStatement, MPPQueryContext context) {
+ super(insertRowsStatement, context);
+ }
+
+ @Override
+ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
+ return visitor.visitInsertRows(this, context);
+ }
+
+ @Override
+ public InsertRowsStatement getInnerTreeStatement() {
+ return ((InsertRowsStatement) super.getInnerTreeStatement());
+ }
+
+ @Override
+ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryProcessException {
+ getInnerTreeStatement().updateAfterSchemaValidation(context);
+ }
+
+ @Override
+ public String getDatabase() {
+ return context.getSession().getDatabaseName().orElse(null);
+ }
+
+ @Override
+ public String getTableName() {
+ return getInnerTreeStatement().getDevicePath().getFullPath();
+ }
+
+ @Override
+ public List<Object[]> getDeviceIdList() {
+ final InsertRowsStatement insertRowStatement = getInnerTreeStatement();
+ return insertRowStatement.getDeviceIdList();
+ }
+
+ @Override
+ public List<String> getAttributeColumnNameList() {
+ // each row may have different columns
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Object[]> getAttributeValueList() {
+ // each row may have different columns
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void validateTableSchema(TableSchema realSchema) {
+ for (InsertRowStatement insertRowStatement :
+ getInnerTreeStatement().getInsertRowStatementList()) {
+ final TableSchema incomingTableSchema = toTableSchema(insertRowStatement);
+ validateTableSchema(realSchema, incomingTableSchema, insertRowStatement);
+ }
+ }
+
+ @Override
+ public void validateDeviceSchema(Metadata metadata, MPPQueryContext context) {
+ for (InsertRowStatement insertRowStatement :
+ getInnerTreeStatement().getInsertRowStatementList()) {
+ metadata.validateDeviceSchema(createTableSchemaValidation(insertRowStatement), context);
+ }
+ }
+
+ protected ITableDeviceSchemaValidation createTableSchemaValidation(
+ InsertRowStatement insertRowStatement) {
+ return new ITableDeviceSchemaValidation() {
+
+ @Override
+ public String getDatabase() {
+ return InsertRows.this.getDatabase();
+ }
+
+ @Override
+ public String getTableName() {
+ return InsertRows.this.getTableName();
+ }
+
+ @Override
+ public List<Object[]> getDeviceIdList() {
+ return Collections.singletonList(insertRowStatement.getTableDeviceID().getSegments());
+ }
+
+ @Override
+ public List<String> getAttributeColumnNameList() {
+ List<String> attributeColumnNameList = new ArrayList<>();
+ for (int i = 0; i < insertRowStatement.getColumnCategories().length; i++) {
+ if (insertRowStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) {
+ attributeColumnNameList.add(insertRowStatement.getMeasurements()[i]);
+ }
+ }
+ return attributeColumnNameList;
+ }
+
+ @Override
+ public List<Object[]> getAttributeValueList() {
+ List<Object> attributeValueList = new ArrayList<>();
+ for (int i = 0; i < insertRowStatement.getColumnCategories().length; i++) {
+ if (insertRowStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) {
+ attributeValueList.add(insertRowStatement.getValues()[i]);
+ }
+ }
+ return Collections.singletonList(attributeValueList.toArray());
+ }
+ };
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
new file mode 100644
index 00000000000..522bd052234
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class InsertTablet extends WrappedInsertStatement {
+
+ public InsertTablet(InsertTabletStatement insertTabletStatement, MPPQueryContext context) {
+ super(insertTabletStatement, context);
+ }
+
+ @Override
+ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
+ return visitor.visitInsertTablet(this, context);
+ }
+
+ @Override
+ public InsertTabletStatement getInnerTreeStatement() {
+ return ((InsertTabletStatement) super.getInnerTreeStatement());
+ }
+
+ @Override
+ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryProcessException {
+ getInnerTreeStatement().updateAfterSchemaValidation(context);
+ }
+
+ @Override
+ public String getDatabase() {
+ return context.getSession().getDatabaseName().get();
+ }
+
+ @Override
+ public String getTableName() {
+ return getInnerTreeStatement().getDevicePath().getFullPath();
+ }
+
+ @Override
+ public List<Object[]> getDeviceIdList() {
+ List<Object[]> deviceIdList = new ArrayList<>();
+ final InsertTabletStatement insertTabletStatement = getInnerTreeStatement();
+ for (int i = 0; i < insertTabletStatement.getRowCount(); i++) {
+ IDeviceID deviceID = insertTabletStatement.getTableDeviceID(i);
+ Object[] deviceIdSegments = new Object[deviceID.segmentNum()];
+ for (int j = 0; j < deviceIdSegments.length; j++) {
+ deviceIdSegments[j] = deviceID.segment(j);
+ }
+ deviceIdList.add(deviceIdSegments);
+ }
+ return deviceIdList;
+ }
+
+ @Override
+ public List<String> getAttributeColumnNameList() {
+ final InsertTabletStatement insertTabletStatement = getInnerTreeStatement();
+ List<String> result = new ArrayList<>();
+ for (int i = 0; i < insertTabletStatement.getColumnCategories().length; i++) {
+ if (insertTabletStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) {
+ result.add(insertTabletStatement.getMeasurements()[i]);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<Object[]> getAttributeValueList() {
+ final InsertTabletStatement insertTabletStatement = getInnerTreeStatement();
+ List<Object[]> result = new ArrayList<>(insertTabletStatement.getRowCount());
+ final List<Integer> attrColumnIndices = insertTabletStatement.getAttrColumnIndices();
+ for (int i = 0; i < insertTabletStatement.getRowCount(); i++) {
+ Object[] attrValues = new Object[attrColumnIndices.size()];
+ for (int j = 0; j < attrColumnIndices.size(); j++) {
+ final int columnIndex = attrColumnIndices.get(j);
+ attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
+ }
+ result.add(attrValues);
+ }
+ return result;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
index 71d130ad228..b60e5a454a7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
@@ -39,4 +39,9 @@ public abstract class Literal extends Expression {
public List<Node> getChildren() {
return ImmutableList.of();
}
+
+ /**
+ * @return a value that can be processed by IoTDB storage engine
+ */
+ public abstract Object getTsValue();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LongLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LongLiteral.java
index bd51f0c6647..fc52071e98c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LongLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LongLiteral.java
@@ -129,4 +129,9 @@ public class LongLiteral extends Literal {
this.value = ReadWriteIOUtils.readString(byteBuffer);
this.parsedValue = ReadWriteIOUtils.readLong(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return value;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/NullLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/NullLiteral.java
index bdf84f0d7a4..2fd8dab0597 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/NullLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/NullLiteral.java
@@ -73,4 +73,9 @@ public class NullLiteral extends Literal {
public NullLiteral(ByteBuffer byteBuffer) {
super(null);
}
+
+ @Override
+ public Object getTsValue() {
+ return null;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
index 1a987f88e08..0bf9c7046fe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
@@ -19,11 +19,13 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
import static java.util.Objects.requireNonNull;
@@ -102,4 +104,9 @@ public class StringLiteral extends Literal {
this.value = ReadWriteIOUtils.readString(byteBuffer);
this.length = ReadWriteIOUtils.readInt(byteBuffer);
}
+
+ @Override
+ public Object getTsValue() {
+ return new Binary(value.getBytes(StandardCharsets.UTF_8));
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
new file mode 100644
index 00000000000..a609d5f5218
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.type.TypeFactory;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
+
+public abstract class WrappedInsertStatement extends WrappedStatement
+ implements ITableDeviceSchemaValidation {
+
+ protected TableSchema tableSchema;
+
+ public WrappedInsertStatement(InsertBaseStatement innerTreeStatement, MPPQueryContext context) {
+ super(innerTreeStatement, context);
+ }
+
+ @Override
+ public InsertBaseStatement getInnerTreeStatement() {
+ return ((InsertBaseStatement) super.getInnerTreeStatement());
+ }
+
+ public abstract void updateAfterSchemaValidation(MPPQueryContext context)
+ throws QueryProcessException;
+
+ public TableSchema getTableSchema() {
+ if (tableSchema == null) {
+ InsertBaseStatement insertBaseStatement = getInnerTreeStatement();
+ tableSchema = toTableSchema(insertBaseStatement);
+ }
+
+ return tableSchema;
+ }
+
+ protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) {
+ String tableName = insertBaseStatement.getDevicePath().getFullPath();
+ List<ColumnSchema> columnSchemas =
+ new ArrayList<>(insertBaseStatement.getMeasurements().length);
+ for (int i = 0; i < insertBaseStatement.getMeasurements().length; i++) {
+ columnSchemas.add(
+ new ColumnSchema(
+ insertBaseStatement.getMeasurements()[i],
+ insertBaseStatement.getDataTypes() != null
+ ? TypeFactory.getType(insertBaseStatement.getDataTypes()[i])
+ : null,
+ false,
+ insertBaseStatement.getColumnCategories() != null
+ ? insertBaseStatement.getColumnCategories()[i]
+ : null));
+ }
+ return new TableSchema(tableName, columnSchemas);
+ }
+
+ public void validateTableSchema(TableSchema realSchema) {
+ final TableSchema incomingTableSchema = getTableSchema();
+ InsertBaseStatement innerTreeStatement = getInnerTreeStatement();
+ validateTableSchema(realSchema, incomingTableSchema, innerTreeStatement);
+ }
+
+ protected void validateTableSchema(
+ TableSchema realSchema,
+ TableSchema incomingTableSchema,
+ InsertBaseStatement innerTreeStatement) {
+ final List<ColumnSchema> incomingSchemaColumns = incomingTableSchema.getColumns();
+ Map<String, ColumnSchema> realSchemaMap = new HashMap<>();
+ realSchema.getColumns().forEach(c -> realSchemaMap.put(c.getName(), c));
+ // incoming schema should be consistent with real schema
+ for (int i = 0, incomingSchemaColumnsSize = incomingSchemaColumns.size();
+ i < incomingSchemaColumnsSize;
+ i++) {
+ ColumnSchema incomingSchemaColumn = incomingSchemaColumns.get(i);
+ final ColumnSchema realSchemaColumn = realSchemaMap.get(incomingSchemaColumn.getName());
+ validateTableSchema(incomingSchemaColumn, realSchemaColumn, i, innerTreeStatement);
+ }
+ // incoming schema should contain all id columns in real schema and have consistent order
+ final List<ColumnSchema> realIdColumns = realSchema.getIdColumns();
+ adjustIdColumns(realIdColumns, innerTreeStatement);
+ }
+
+ /**
+ * Adjust the order of ID columns in this insertion to be consistent with that from the schema
+ * region.
+ *
+ * @param realIdColumnSchemas id column order from the schema region
+ */
+ public void adjustIdColumns(
+ List<ColumnSchema> realIdColumnSchemas, final InsertBaseStatement baseStatement) {
+ List<ColumnSchema> incomingColumnSchemas = getTableSchema().getColumns();
+ for (int realIdColPos = 0; realIdColPos < realIdColumnSchemas.size(); realIdColPos++) {
+ ColumnSchema realColumn = realIdColumnSchemas.get(realIdColPos);
+ int incomingIdColPos = incomingColumnSchemas.indexOf(realColumn);
+ if (incomingIdColPos == -1) {
+ // if the realIdColPos-th id column in the table is missing, insert an empty column in the
+ // tablet
+ baseStatement.insertColumn(realIdColPos, realColumn);
+ incomingColumnSchemas.add(realIdColPos, realColumn);
+ } else {
+ // move the id column in the tablet to the proper position
+ baseStatement.swapColumn(incomingIdColPos, realIdColPos);
+ Collections.swap(incomingColumnSchemas, incomingIdColPos, realIdColPos);
+ }
+ }
+ tableSchema = null;
+ }
+
+ public static void validateTableSchema(
+ ColumnSchema incoming, ColumnSchema real, int i, InsertBaseStatement innerTreeStatement) {
+ if (real == null) {
+ // the column does not exist and auto-creation is disabled
+ SemanticException semanticException =
+ new SemanticException(
+ "Column " + incoming.getName() + " does not exists or fails to be " + "created");
+ if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) {
+ // non-measurement columns cannot be partially inserted
+ throw semanticException;
+ } else {
+ // partial insertion
+ innerTreeStatement.markFailedMeasurement(i, semanticException);
+ return;
+ }
+ }
+ if (incoming.getType() == null) {
+ // sql insertion does not provide type
+ innerTreeStatement.setDataType(InternalTypeManager.getTSDataType(real.getType()), i);
+ } else if (!incoming.getType().equals(real.getType())) {
+ SemanticException semanticException =
+ new SemanticException(
+ String.format(
+ "Inconsistent data type of column %s: %s/%s",
+ incoming.getName(), incoming.getType(), real.getType()));
+ if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) {
+ // non-measurement columns cannot be partially inserted
+ throw semanticException;
+ } else {
+ // partial insertion
+ innerTreeStatement.markFailedMeasurement(i, semanticException);
+ }
+ }
+ if (incoming.getColumnCategory() == null) {
+ // sql insertion does not provide category
+ innerTreeStatement.setColumnCategory(real.getColumnCategory(), i);
+ } else if (!incoming.getColumnCategory().equals(real.getColumnCategory())) {
+ throw new SemanticException(
+ String.format(
+ "Inconsistent column category of column %s: %s/%s",
+ incoming.getName(), incoming.getColumnCategory(), real.getColumnCategory()));
+ }
+ TSDataType tsDataType = InternalTypeManager.getTSDataType(real.getType());
+ MeasurementSchema measurementSchema =
+ new MeasurementSchema(
+ real.getName(),
+ tsDataType,
+ getDefaultEncoding(tsDataType),
+ TSFileDescriptor.getInstance().getConfig().getCompressor());
+ innerTreeStatement.setMeasurementSchema(measurementSchema, i);
+ }
+
+ public void validateDeviceSchema(Metadata metadata, MPPQueryContext context) {
+ metadata.validateDeviceSchema(this, context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedStatement.java
new file mode 100644
index 00000000000..003dd2ed431
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedStatement.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public abstract class WrappedStatement extends Statement {
+ protected org.apache.iotdb.db.queryengine.plan.statement.Statement innerTreeStatement;
+ protected MPPQueryContext context;
+
+ public WrappedStatement(
+ org.apache.iotdb.db.queryengine.plan.statement.Statement innerTreeStatement,
+ MPPQueryContext context) {
+ super(null);
+ this.innerTreeStatement = innerTreeStatement;
+ this.context = context;
+ }
+
+ @Override
+ public List<? extends Node> getChildren() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public int hashCode() {
+ return innerTreeStatement.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ WrappedStatement that = (WrappedStatement) o;
+ return Objects.equals(innerTreeStatement, that.innerTreeStatement);
+ }
+
+ @Override
+ public String toString() {
+ return innerTreeStatement.toString();
+ }
+
+ public org.apache.iotdb.db.queryengine.plan.statement.Statement getInnerTreeStatement() {
+ return innerTreeStatement;
+ }
+
+ public void setInnerTreeStatement(
+ org.apache.iotdb.db.queryengine.plan.statement.Statement innerTreeStatement) {
+ this.innerTreeStatement = innerTreeStatement;
+ }
+
+ public MPPQueryContext getContext() {
+ return context;
+ }
+
+ public void setContext(MPPQueryContext context) {
+ this.context = context;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
index f69ba470c95..02ca1c4a6f0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation;
@@ -58,6 +59,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupBy;
@@ -69,6 +71,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNotNullPredicate;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNullPredicate;
@@ -131,6 +134,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlBaseVisitor;
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlLexer;
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlParser;
@@ -158,6 +166,7 @@ import java.util.function.Function;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
+import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.ATTRIBUTE;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.ID;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT;
@@ -336,15 +345,88 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> {
@Override
public Node visitInsertStatement(RelationalSqlParser.InsertStatementContext ctx) {
if (ctx.columnAliases() != null) {
- return new Insert(
- new Table(getQualifiedName(ctx.tableName)),
- visit(ctx.columnAliases().identifier(), Identifier.class),
- (Query) visit(ctx.query()));
+ Query query = (Query) visit(ctx.query());
+ List<Identifier> identifiers = visit(ctx.columnAliases().identifier(), Identifier.class);
+ if (query.getQueryBody() instanceof Values) {
+ return visitInsertValues(
+ getQualifiedName(ctx.tableName), identifiers, ((Values) query.getQueryBody()));
+ }
+ return new Insert(new Table(getQualifiedName(ctx.tableName)), identifiers, query);
} else {
return new Insert(new Table(getQualifiedName(ctx.tableName)), (Query) visit(ctx.query()));
}
}
+ private Node visitInsertValues(
+ QualifiedName tableName, List<Identifier> identifiers, Values queryBody) {
+ String tableNameString = tableName.toString();
+ List<String> columnNames = identifiers.stream().map(Identifier::getValue).collect(toList());
+ int timeColumnIndex = -1;
+ for (int i = 0; i < columnNames.size(); i++) {
+ if (TIME_COLUMN_NAME.equalsIgnoreCase(columnNames.get(i))) {
+ timeColumnIndex = i;
+ break;
+ }
+ }
+ if (timeColumnIndex != -1) {
+ columnNames.remove(timeColumnIndex);
+ }
+ String[] columnNameArray = columnNames.toArray(new String[0]);
+
+ List<Expression> rows = queryBody.getRows();
+ int finalTimeColumnIndex = timeColumnIndex;
+ List<InsertRowStatement> rowStatements =
+ rows.stream()
+ // Row -> List<Expression>
+ .map(
+ r ->
+ toInsertRowStatement(
+ ((Row) r), finalTimeColumnIndex, columnNameArray, tableNameString))
+ .collect(toList());
+
+ InsertRowsStatement insertRowsStatement = new InsertRowsStatement();
+ insertRowsStatement.setInsertRowStatementList(rowStatements);
+ insertRowsStatement.setWriteToTable(true);
+ insertRowsStatement.setDevicePath(new PartialPath(new String[] {tableNameString}));
+ insertRowsStatement.setMeasurements(columnNameArray);
+ return new InsertRows(insertRowsStatement, null);
+ }
+
+ private InsertRowStatement toInsertRowStatement(
+ Row row, int timeColumnIndex, String[] columnNames, String tableName) {
+ InsertRowStatement insertRowStatement = new InsertRowStatement();
+ insertRowStatement.setWriteToTable(true);
+ insertRowStatement.setDevicePath(new PartialPath(new String[] {tableName}));
+ long timestamp;
+ if (timeColumnIndex == -1) {
+ timestamp = CommonDateTimeUtils.currentTime();
+ } else {
+ Expression timeExpression = row.getItems().get(timeColumnIndex);
+ if (timeExpression instanceof LongLiteral) {
+ timestamp = ((LongLiteral) timeExpression).getParsedValue();
+ } else {
+ timestamp =
+ parseDateTimeFormat(
+ ((StringLiteral) timeExpression).getValue(),
+ CommonDateTimeUtils.currentTime(),
+ zoneId);
+ }
+ }
+ insertRowStatement.setTime(timestamp);
+ insertRowStatement.setMeasurements(columnNames);
+
+ Object[] values = new Object[columnNames.length];
+ int valuePos = 0;
+ for (int i = 0; i < row.getItems().size(); i++) {
+ if (i != timeColumnIndex) {
+ values[valuePos++] = AstUtil.expressionToTsValue(row.getItems().get(i));
+ }
+ }
+ insertRowStatement.setValues(values);
+ insertRowStatement.setNeedInferType(true);
+ return insertRowStatement;
+ }
+
@Override
public Node visitDeleteStatement(RelationalSqlParser.DeleteStatementContext ctx) {
if (ctx.booleanExpression() != null) {
@@ -497,7 +579,26 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> {
@Override
public Node visitFlushStatement(RelationalSqlParser.FlushStatementContext ctx) {
- return super.visitFlushStatement(ctx);
+ FlushStatement flushStatement = new FlushStatement(StatementType.FLUSH);
+ List<PartialPath> storageGroups = null;
+ if (ctx.booleanValue() != null) {
+ flushStatement.setSeq(Boolean.parseBoolean(ctx.booleanValue().getText()));
+ }
+ flushStatement.setOnCluster(
+ ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null);
+ if (ctx.identifier() != null) {
+ storageGroups = new ArrayList<>();
+ List<Identifier> identifiers = getIdentifiers(ctx.identifier());
+ for (Identifier identifier : identifiers) {
+ try {
+ storageGroups.add(new PartialPath(PathUtils.qualifyDatabaseName(identifier.getValue())));
+ } catch (IllegalPathException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ flushStatement.setStorageGroups(storageGroups);
+ return new Flush(flushStatement, null);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
index 09ef3e18720..666ee1437c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
@@ -19,6 +19,9 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.util;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
import com.google.common.graph.SuccessorsFunction;
@@ -95,5 +98,12 @@ public final class AstUtil {
return result;
}
+ public static Object expressionToTsValue(Expression expression) {
+ if (expression instanceof Literal) {
+ return ((Literal) expression).getTsValue();
+ }
+ throw new SemanticException("Unsupported expression: " + expression);
+ }
+
private AstUtil() {}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/ConversionUtils.java
similarity index 77%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/ConversionUtils.java
index 2e17d0775d9..48aaa6fef40 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/sql/SemanticException.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/ConversionUtils.java
@@ -17,15 +17,10 @@
* under the License.
*/
-package org.apache.iotdb.db.exception.sql;
+package org.apache.iotdb.db.queryengine.plan.relational.utils;
-public class SemanticException extends RuntimeException {
-
- public SemanticException(String message) {
- super(message);
- }
-
- public SemanticException(Throwable cause) {
- super(cause);
+public class ConversionUtils {
+ private ConversionUtils() {
+ // util class
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/Statement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/Statement.java
index f74d210c667..d7aac386a88 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/Statement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/Statement.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.statement;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.auth.AuthorityChecker;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor;
import java.util.List;
@@ -66,4 +67,9 @@ public abstract class Statement extends StatementNode {
AuthorityChecker.SUPER_USER.equals(userName),
"Only the admin user can perform this operation");
}
+
+ public org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement toRelationalStatement(
+ MPPQueryContext context) {
+ throw new UnsupportedOperationException("Method not implemented yet");
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
index bc31e71ec80..027cfba0c86 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.auth.entity.PrivilegeType;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.schema.view.LogicalViewSchema;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -32,7 +33,10 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
@@ -68,6 +72,11 @@ public abstract class InsertBaseStatement extends Statement {
/** index of failed measurements -> info including measurement, data type and value */
protected Map<Integer, FailedMeasurementInfo> failedMeasurementIndex2Info;
+ protected TsTableColumnCategory[] columnCategories;
+ protected List<Integer> idColumnIndices;
+ protected List<Integer> attrColumnIndices;
+ protected boolean writeToTable = false;
+
// region params used by analyzing logical views.
/** This param records the logical view schema appeared in this statement. */
@@ -113,6 +122,13 @@ public abstract class InsertBaseStatement extends Statement {
this.measurementSchemas = measurementSchemas;
}
+ public void setMeasurementSchema(MeasurementSchema measurementSchema, int i) {
+ if (measurementSchemas == null) {
+ measurementSchemas = new MeasurementSchema[measurements.length];
+ }
+ measurementSchemas[i] = measurementSchema;
+ }
+
public boolean isAligned() {
return isAligned;
}
@@ -129,6 +145,13 @@ public abstract class InsertBaseStatement extends Statement {
this.dataTypes = dataTypes;
}
+ public void setDataType(TSDataType dataType, int i) {
+ if (dataTypes == null) {
+ dataTypes = new TSDataType[measurements.length];
+ }
+ this.dataTypes[i] = dataType;
+ }
+
/** Returns true when this statement is empty and no need to write into the server */
public abstract boolean isEmpty();
@@ -238,6 +261,46 @@ public abstract class InsertBaseStatement extends Statement {
return false;
}
+ public TsTableColumnCategory[] getColumnCategories() {
+ return columnCategories;
+ }
+
+ public void setColumnCategories(TsTableColumnCategory[] columnCategories) {
+ this.columnCategories = columnCategories;
+ }
+
+ public void setColumnCategory(TsTableColumnCategory columnCategory, int i) {
+ if (columnCategories == null) {
+ columnCategories = new TsTableColumnCategory[measurements.length];
+ }
+ this.columnCategories[i] = columnCategory;
+ this.idColumnIndices = null;
+ }
+
+ public List<Integer> getIdColumnIndices() {
+ if (idColumnIndices == null && columnCategories != null) {
+ idColumnIndices = new ArrayList<>();
+ for (int i = 0; i < columnCategories.length; i++) {
+ if (columnCategories[i].equals(TsTableColumnCategory.ID)) {
+ idColumnIndices.add(i);
+ }
+ }
+ }
+ return idColumnIndices;
+ }
+
+ public List<Integer> getAttrColumnIndices() {
+ if (attrColumnIndices == null && columnCategories != null) {
+ attrColumnIndices = new ArrayList<>();
+ for (int i = 0; i < columnCategories.length; i++) {
+ if (columnCategories[i].equals(TsTableColumnCategory.ATTRIBUTE)) {
+ attrColumnIndices.add(i);
+ }
+ }
+ }
+ return attrColumnIndices;
+ }
+
public boolean hasFailedMeasurements() {
return failedMeasurementIndex2Info != null && !failedMeasurementIndex2Info.isEmpty();
}
@@ -383,5 +446,83 @@ public abstract class InsertBaseStatement extends Statement {
}
}
}
+
+ public void insertColumn(int pos, ColumnSchema columnSchema) {
+ if (pos < 0 || pos > measurements.length) {
+ throw new ArrayIndexOutOfBoundsException(pos);
+ }
+
+ if (measurementSchemas != null) {
+ MeasurementSchema[] tmp = new MeasurementSchema[measurementSchemas.length + 1];
+ System.arraycopy(measurementSchemas, 0, tmp, 0, pos);
+ tmp[pos] =
+ new MeasurementSchema(
+ columnSchema.getName(), InternalTypeManager.getTSDataType(columnSchema.getType()));
+ System.arraycopy(measurementSchemas, pos, tmp, pos + 1, measurementSchemas.length - pos);
+ measurementSchemas = tmp;
+ }
+
+ String[] tmpMeasurements = new String[measurements.length + 1];
+ System.arraycopy(measurements, 0, tmpMeasurements, 0, pos);
+ tmpMeasurements[pos] = columnSchema.getName();
+ System.arraycopy(measurements, pos, tmpMeasurements, pos + 1, measurements.length - pos);
+ measurements = tmpMeasurements;
+
+ if (dataTypes == null) {
+ // sql insertion
+ dataTypes = new TSDataType[measurements.length + 1];
+ dataTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType());
+ } else {
+ TSDataType[] tmpTypes = new TSDataType[dataTypes.length + 1];
+ System.arraycopy(dataTypes, 0, tmpTypes, 0, pos);
+ tmpTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType());
+ System.arraycopy(dataTypes, pos, tmpTypes, pos + 1, dataTypes.length - pos);
+ dataTypes = tmpTypes;
+ }
+
+ if (columnCategories == null) {
+ columnCategories = new TsTableColumnCategory[measurements.length + 1];
+ columnCategories[pos] = columnSchema.getColumnCategory();
+ } else {
+ TsTableColumnCategory[] tmpCategories =
+ new TsTableColumnCategory[columnCategories.length + 1];
+ System.arraycopy(columnCategories, 0, tmpCategories, 0, pos);
+ tmpCategories[pos] = columnSchema.getColumnCategory();
+ System.arraycopy(
+ columnCategories, pos, tmpCategories, pos + 1, columnCategories.length - pos);
+ columnCategories = tmpCategories;
+ idColumnIndices = null;
+ }
+ }
+
+ public void swapColumn(int src, int target) {
+ if (src < 0 || src >= measurements.length || target < 0 || target >= measurements.length) {
+ throw new ArrayIndexOutOfBoundsException(src + "/" + target);
+ }
+ if (measurementSchemas != null) {
+ CommonUtils.swapArray(measurementSchemas, src, target);
+ }
+ CommonUtils.swapArray(measurements, src, target);
+ // dataTypes is null for sql insertion
+ if (dataTypes != null) {
+ CommonUtils.swapArray(dataTypes, src, target);
+ }
+ if (columnCategories != null) {
+ CommonUtils.swapArray(columnCategories, src, target);
+ }
+ idColumnIndices = null;
+ }
+
+ public boolean isWriteToTable() {
+ return writeToTable;
+ }
+
+ public void setWriteToTable(boolean writeToTable) {
+ this.writeToTable = writeToTable;
+ if (writeToTable) {
+ isAligned = true;
+ }
+ }
+
// endregion
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
index a5b864e30f4..a07df1a3693 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
@@ -32,12 +32,17 @@ import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.Pair;
@@ -72,6 +77,8 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa
*/
private boolean[] measurementIsAligned;
+ private IDeviceID deviceID;
+
public InsertRowStatement() {
super();
statementType = StatementType.INSERT;
@@ -221,7 +228,9 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa
// parse string value to specific type
dataTypes[i] = measurementSchemas[i].getType();
try {
- values[i] = CommonUtils.parseValue(dataTypes[i], values[i].toString(), zoneId);
+ if (values[i] != null) {
+ values[i] = CommonUtils.parseValue(dataTypes[i], values[i].toString(), zoneId);
+ }
} catch (Exception e) {
LOGGER.warn(
"data type of {}.{} is not consistent, "
@@ -444,4 +453,39 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa
return new Pair<>(
this.recordedBeginOfLogicalViewSchemaList, this.recordedEndOfLogicalViewSchemaList);
}
+
+ public IDeviceID getTableDeviceID() {
+ if (deviceID == null) {
+ String[] deviceIdSegments = new String[getIdColumnIndices().size() + 1];
+ deviceIdSegments[0] = this.devicePath.getFullPath();
+ for (int i = 0; i < getIdColumnIndices().size(); i++) {
+ final Integer columnIndex = getIdColumnIndices().get(i);
+ deviceIdSegments[i + 1] =
+ values[columnIndex] != null ? values[columnIndex].toString() : null;
+ }
+ deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+
+ return deviceID;
+ }
+
+ @Override
+ public Statement toRelationalStatement(MPPQueryContext context) {
+ return new InsertRow(this, context);
+ }
+
+ @Override
+ public void insertColumn(int pos, ColumnSchema columnSchema) {
+ super.insertColumn(pos, columnSchema);
+ Object[] tmpValues = new Object[values.length + 1];
+ System.arraycopy(values, 0, tmpValues, 0, pos);
+ System.arraycopy(values, pos, tmpValues, pos + 1, values.length - pos);
+ values = tmpValues;
+ }
+
+ @Override
+ public void swapColumn(int src, int target) {
+ super.swapColumn(src, target);
+ CommonUtils.swapArray(values, src, target);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
index 5dbdba2f228..aa448c6613e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
@@ -162,4 +162,10 @@ public class InsertRowsStatement extends InsertBaseStatement {
splitResult.setInsertRowStatementList(mergedList);
return splitResult;
}
+
+ public List<Object[]> getDeviceIdList() {
+ return insertRowStatementList.stream()
+ .map(s -> s.getTableDeviceID().getSegments())
+ .collect(Collectors.toList());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index b38997ef9c0..ce3fb28f913 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -26,13 +26,22 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
+import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.Binary;
@@ -60,6 +69,8 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
private BitMap[] bitMaps;
private Object[] columns;
+ private IDeviceID[] deviceIDs;
+
private int rowCount = 0;
/**
@@ -75,6 +86,25 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
this.recordedEndOfLogicalViewSchemaList = 0;
}
+ public InsertTabletStatement(InsertTabletNode node) {
+ this();
+ setDevicePath(node.getDevicePath());
+ setMeasurements(node.getMeasurements());
+ setTimes(node.getTimes());
+ setColumns(node.getColumns());
+ setBitMaps(node.getBitMaps());
+ setRowCount(node.getRowCount());
+ setDataTypes(node.getDataTypes());
+ setAligned(node.isAligned());
+ setMeasurementSchemas(node.getMeasurementSchemas());
+ }
+
+ public InsertTabletStatement(RelationalInsertTabletNode node) {
+ this(((InsertTabletNode) node));
+ setColumnCategories(node.getColumnCategories());
+ setWriteToTable(true);
+ }
+
public int getRowCount() {
return rowCount;
}
@@ -132,6 +162,10 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
return result;
}
+ public TTimePartitionSlot getTimePartitionSlot(int i) {
+ return TimePartitionUtils.getTimePartitionSlot(times[i]);
+ }
+
@Override
public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
return visitor.visitInsertTablet(this, context);
@@ -404,4 +438,65 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
return new Pair<>(
this.recordedBeginOfLogicalViewSchemaList, this.recordedEndOfLogicalViewSchemaList);
}
+
+ @Override
+ public Statement toRelationalStatement(MPPQueryContext context) {
+ return new InsertTablet(this, context);
+ }
+
+ public IDeviceID getTableDeviceID(int rowIdx) {
+ if (deviceIDs == null) {
+ deviceIDs = new IDeviceID[rowCount];
+ }
+ if (deviceIDs[rowIdx] == null) {
+ String[] deviceIdSegments = new String[getIdColumnIndices().size() + 1];
+ deviceIdSegments[0] = this.devicePath.getFullPath();
+ for (int i = 0; i < getIdColumnIndices().size(); i++) {
+ final Integer columnIndex = getIdColumnIndices().get(i);
+ Object idSeg = ((Object[]) columns[columnIndex])[rowIdx];
+ deviceIdSegments[i + 1] = idSeg != null ? idSeg.toString() : null;
+ }
+ deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+
+ return deviceIDs[rowIdx];
+ }
+
+ @Override
+ public void insertColumn(int pos, ColumnSchema columnSchema) {
+ super.insertColumn(pos, columnSchema);
+
+ if (bitMaps != null) {
+ BitMap[] tmpBitmaps = new BitMap[bitMaps.length + 1];
+ System.arraycopy(bitMaps, 0, tmpBitmaps, 0, pos);
+ tmpBitmaps[pos] = new BitMap(rowCount);
+ for (int i = 0; i < rowCount; i++) {
+ tmpBitmaps[pos].mark(i);
+ }
+ System.arraycopy(bitMaps, pos, tmpBitmaps, pos + 1, bitMaps.length - pos);
+ bitMaps = tmpBitmaps;
+ }
+
+ Object[] tmpColumns = new Object[columns.length + 1];
+ System.arraycopy(columns, 0, tmpColumns, 0, pos);
+ tmpColumns[pos] =
+ CommonUtils.createValueColumnOfDataType(
+ InternalTypeManager.getTSDataType(columnSchema.getType()),
+ columnSchema.getColumnCategory(),
+ rowCount);
+ System.arraycopy(columns, pos, tmpColumns, pos + 1, columns.length - pos);
+ columns = tmpColumns;
+
+ deviceIDs = null;
+ }
+
+ @Override
+ public void swapColumn(int src, int target) {
+ super.swapColumn(src, target);
+ if (bitMaps != null) {
+ CommonUtils.swapArray(bitMaps, src, target);
+ }
+ CommonUtils.swapArray(columns, src, target);
+ deviceIDs = null;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
index 457b48e0a6c..38290b8aa4b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
@@ -25,6 +25,8 @@ import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,7 +125,10 @@ public class DeviceAttributeStore implements IDeviceAttributeStore {
Map<String, String> attributeMap = new HashMap<>();
String value;
for (int i = 0; i < nameList.size(); i++) {
- value = valueList[i] == null ? null : (String) valueList[i];
+ value =
+ valueList[i] == null
+ ? null
+ : ((Binary) valueList[i]).getStringValue(TSFileConfig.STRING_CHARSET);
if (value != null) {
attributeMap.put(nameList.get(i), value);
memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value);
@@ -144,7 +149,7 @@ public class DeviceAttributeStore implements IDeviceAttributeStore {
String value;
for (int i = 0; i < nameList.size(); i++) {
String key = nameList.get(i);
- value = valueList[i] == null ? null : (String) valueList[i];
+ value = valueList[i] == null ? null : valueList[i].toString();
if (value != null) {
originMemUsage =
attributeMap.containsKey(key)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 22fc44c4b43..bc0fe7581a7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -80,6 +80,8 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -123,6 +125,8 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
*/
public class MTreeBelowSGMemoryImpl {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MTreeBelowSGMemoryImpl.class);
+
// this implementation is based on memory, thus only MTree write operation must invoke MTreeStore
private final MemMTreeStore store;
@@ -1185,6 +1189,7 @@ public class MTreeBelowSGMemoryImpl {
while (deviceIdIterator.hasNext()) {
try {
IMemMNode node = getTableDeviceNode(table, deviceIdIterator.next());
+
if (!node.isDevice()) {
continue;
}
@@ -1502,19 +1507,13 @@ public class MTreeBelowSGMemoryImpl {
IntConsumer attributeUppdater)
throws MetadataException {
// todo implement storage for device of diverse data types
- IMemMNode cur = storageGroupMNode;
- IMemMNode child = cur.getChild(tableName);
- if (child == null) {
- child = store.addChild(cur, tableName, nodeFactory.createInternalMNode(cur, tableName));
- }
- cur = child;
- String nodeName;
+ IMemMNode cur = storageGroupMNode;
for (Object o : devicePath) {
- nodeName = o == null ? null : (String) o;
- child = cur.getChild(nodeName);
+ String childName = o == null ? null : o.toString();
+ IMemMNode child = cur.getChild(childName);
if (child == null) {
- child = store.addChild(cur, nodeName, nodeFactory.createInternalMNode(cur, nodeName));
+ child = store.addChild(cur, childName, nodeFactory.createInternalMNode(cur, childName));
}
cur = child;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
index 60413cc579f..0ab7fc37fab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.schemaengine.table;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -94,7 +95,8 @@ public class DataNodeTableCache implements ITableCache {
}
@Override
- public void preUpdateTable(final String database, final TsTable table) {
+ public void preUpdateTable(String database, final TsTable table) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
preUpdateTableMap
@@ -107,7 +109,8 @@ public class DataNodeTableCache implements ITableCache {
}
@Override
- public void rollbackUpdateTable(final String database, final String tableName) {
+ public void rollbackUpdateTable(String database, final String tableName) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
removeTableFromPreUpdateMap(database, tableName);
@@ -134,7 +137,8 @@ public class DataNodeTableCache implements ITableCache {
}
@Override
- public void commitUpdateTable(final String database, final String tableName) {
+ public void commitUpdateTable(String database, final String tableName) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
final TsTable table = preUpdateTableMap.get(database).get(tableName);
@@ -149,7 +153,8 @@ public class DataNodeTableCache implements ITableCache {
}
@Override
- public void invalid(final String database) {
+ public void invalid(String database) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
databaseTableMap.remove(database);
@@ -159,7 +164,8 @@ public class DataNodeTableCache implements ITableCache {
}
}
- public TsTable getTable(final String database, final String tableName) {
+ public TsTable getTable(String database, final String tableName) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.readLock().lock();
try {
if (databaseTableMap.containsKey(database)) {
@@ -171,7 +177,8 @@ public class DataNodeTableCache implements ITableCache {
}
}
- public Optional<List<TsTable>> getTables(final String database) {
+ public Optional<List<TsTable>> getTables(String database) {
+ database = PathUtils.qualifyDatabaseName(database);
readWriteLock.readLock().lock();
try {
final Map<String, TsTable> tableMap = databaseTableMap.get(database);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8ad2b6c8c0c..316dd3329c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.SchemaConstant;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
@@ -56,10 +57,13 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCach
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema;
+import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.db.service.SettleService;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
@@ -111,6 +115,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.TimePartitionManager;
import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager;
import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager;
import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool;
+import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -158,6 +163,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.IntToLongFunction;
import java.util.stream.Collectors;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
@@ -394,6 +400,7 @@ public class DataRegion implements IDataRegionForQuery {
/** this class is used to store recovering context. */
private class DataRegionRecoveryContext {
+
/** number of files to be recovered. */
private final long numOfFilesToRecover;
@@ -875,7 +882,7 @@ public class DataRegion implements IDataRegionForQuery {
// reject insertions that are out of ttl
long deviceTTL =
DataNodeTTLCache.getInstance().getTTL(insertRowNode.getDevicePath().getNodes());
- if (!isAlive(insertRowNode.getTime(), deviceTTL)) {
+ if (!CommonUtils.isAlive(insertRowNode.getTime(), deviceTTL)) {
throw new OutOfTTLException(
insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - deviceTTL));
}
@@ -889,18 +896,7 @@ public class DataRegion implements IDataRegionForQuery {
}
// init map
long timePartitionId = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime());
-
- if (config.isEnableSeparateData()
- && !lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionId)) {
- TimePartitionManager.getInstance()
- .registerTimePartitionInfo(
- new TimePartitionInfo(
- new DataRegionId(Integer.parseInt(dataRegionId)),
- timePartitionId,
- true,
- Long.MAX_VALUE,
- 0));
- }
+ initFlushTimeMap(timePartitionId);
boolean isSequence =
config.isEnableSeparateData()
@@ -930,8 +926,91 @@ public class DataRegion implements IDataRegionForQuery {
}
}
+ private long getLastFlushTime(long timePartitionID, IDeviceID deviceID) {
+ return config.isEnableSeparateData()
+ ? lastFlushTimeMap.getFlushedTime(timePartitionID, deviceID)
+ : Long.MAX_VALUE;
+ }
+
+ private boolean splitAndInsert(InsertTabletNode insertTabletNode, int loc, TSStatus[] results)
+ throws BatchProcessException, WriteProcessException {
+ boolean noFailure = true;
+
+ // before is first start point
+ int before = loc;
+ long beforeTime = insertTabletNode.getTimes()[before];
+ // before time partition
+ long beforeTimePartition = TimePartitionUtils.getTimePartitionId(beforeTime);
+ // init flush time map
+ initFlushTimeMap(beforeTimePartition);
+
+ // if is sequence
+ boolean isSequence = false;
+ while (loc < insertTabletNode.getRowCount()) {
+ long time = insertTabletNode.getTimes()[loc];
+ final long timePartitionId = TimePartitionUtils.getTimePartitionId(time);
+
+ long lastFlushTime;
+ // judge if we should insert sequence
+ if (timePartitionId != beforeTimePartition) {
+ initFlushTimeMap(timePartitionId);
+ lastFlushTime = getLastFlushTime(timePartitionId, insertTabletNode.getDeviceID(loc));
+ // a new partition, insert the remaining of the previous partition
+ noFailure =
+ insertTabletToTsFileProcessor(
+ insertTabletNode,
+ before,
+ loc,
+ isSequence,
+ results,
+ beforeTimePartition,
+ noFailure)
+ && noFailure;
+ before = loc;
+ beforeTimePartition = timePartitionId;
+ isSequence = time > lastFlushTime;
+ } else if (!isSequence) {
+ lastFlushTime = getLastFlushTime(timePartitionId, insertTabletNode.getDeviceID(loc));
+ if (time > lastFlushTime) {
+ // the same partition and switch to sequence data
+ // insert previous range into unsequence
+ noFailure =
+ insertTabletToTsFileProcessor(
+ insertTabletNode,
+ before,
+ loc,
+ isSequence,
+ results,
+ beforeTimePartition,
+ noFailure)
+ && noFailure;
+ before = loc;
+ isSequence = true;
+ }
+ }
+ // else: the same partition and isSequence not changed, just move the cursor forward
+ loc++;
+ }
+
+ // do not forget last part
+ if (before < loc) {
+ noFailure =
+ insertTabletToTsFileProcessor(
+ insertTabletNode,
+ before,
+ loc,
+ isSequence,
+ results,
+ beforeTimePartition,
+ noFailure)
+ && noFailure;
+ }
+
+ return noFailure;
+ }
+
/**
- * Insert a tablet (rows belonging to the same devices) into this database.
+ * Insert a tablet into this database.
*
* @throws BatchProcessException if some of the rows failed to be inserted
*/
@@ -950,87 +1029,13 @@ public class DataRegion implements IDataRegionForQuery {
}
TSStatus[] results = new TSStatus[insertTabletNode.getRowCount()];
Arrays.fill(results, RpcUtils.SUCCESS_STATUS);
- boolean noFailure = true;
- long deviceTTL =
- DataNodeTTLCache.getInstance().getTTL(insertTabletNode.getDevicePath().getNodes());
-
- /*
- * assume that batch has been sorted by client
- */
- int loc = 0;
- while (loc < insertTabletNode.getRowCount()) {
- long currTime = insertTabletNode.getTimes()[loc];
- // skip points that do not satisfy TTL
- if (!isAlive(currTime, deviceTTL)) {
- results[loc] =
- RpcUtils.getStatus(
- TSStatusCode.OUT_OF_TTL,
- String.format(
- "Insertion time [%s] is less than ttl time bound [%s]",
- DateTimeUtils.convertLongToDate(currTime),
- DateTimeUtils.convertLongToDate(
- CommonDateTimeUtils.currentTime() - deviceTTL)));
- loc++;
- noFailure = false;
- } else {
- break;
- }
- }
- // loc pointing at first legal position
- if (loc == insertTabletNode.getRowCount()) {
- throw new OutOfTTLException(
- insertTabletNode.getTimes()[insertTabletNode.getTimes().length - 1],
- (CommonDateTimeUtils.currentTime() - deviceTTL));
- }
- // before is first start point
- int before = loc;
- // before time partition
- long beforeTimePartition =
- TimePartitionUtils.getTimePartitionId(insertTabletNode.getTimes()[before]);
- // init map
-
- if (config.isEnableSeparateData()
- && !lastFlushTimeMap.checkAndCreateFlushedTimePartition(beforeTimePartition)) {
- TimePartitionManager.getInstance()
- .registerTimePartitionInfo(
- new TimePartitionInfo(
- new DataRegionId(Integer.parseInt(dataRegionId)),
- beforeTimePartition,
- true,
- Long.MAX_VALUE,
- 0));
- }
+ boolean noFailure;
+ int loc =
+ insertTabletNode.checkTTL(
+ results, i -> DataNodeTTLCache.getInstance().getTTL(insertTabletNode.getDeviceID(i)));
+ noFailure = loc == 0;
- long lastFlushTime =
- config.isEnableSeparateData()
- ? lastFlushTimeMap.getFlushedTime(beforeTimePartition, insertTabletNode.getDeviceID())
- : Long.MAX_VALUE;
-
- // if is sequence
- boolean isSequence = false;
- while (loc < insertTabletNode.getRowCount()) {
- long time = insertTabletNode.getTimes()[loc];
- // always in some time partition
- // judge if we should insert sequence
- if (!isSequence && time > lastFlushTime) {
- // insert into unsequence and then start sequence
- noFailure =
- insertTabletToTsFileProcessor(
- insertTabletNode, before, loc, false, results, beforeTimePartition)
- && noFailure;
- before = loc;
- isSequence = true;
- }
- loc++;
- }
-
- // do not forget last part
- if (before < loc) {
- noFailure =
- insertTabletToTsFileProcessor(
- insertTabletNode, before, loc, isSequence, results, beforeTimePartition)
- && noFailure;
- }
+ noFailure = noFailure && splitAndInsert(insertTabletNode, loc, results);
if (CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) {
if (!insertTabletNode.isGeneratedByRemoteConsensusLeader()) {
@@ -1050,13 +1055,63 @@ public class DataRegion implements IDataRegionForQuery {
}
}
- /**
- * Check whether the time falls in TTL.
- *
- * @return whether the given time falls in ttl
- */
- private boolean isAlive(long time, long dataTTL) {
- return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL;
+ private void initFlushTimeMap(long timePartitionId) {
+ if (config.isEnableSeparateData()
+ && !lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionId)) {
+ TimePartitionManager.getInstance()
+ .registerTimePartitionInfo(
+ new TimePartitionInfo(
+ new DataRegionId(Integer.parseInt(dataRegionId)),
+ timePartitionId,
+ true,
+ Long.MAX_VALUE,
+ 0));
+ }
+ }
+
+ private int checkTTL(
+ InsertTabletNode insertTabletNode,
+ TSStatus[] results,
+ IntToLongFunction rowTTLGetter,
+ boolean breakOnFirstAlive)
+ throws OutOfTTLException {
+
+ /*
+ * assume that batch has been sorted by client
+ */
+ int loc = 0;
+ long ttl = 0;
+ int firstAliveLoc = -1;
+ while (loc < insertTabletNode.getRowCount()) {
+ ttl = rowTTLGetter.applyAsLong(loc);
+ long currTime = insertTabletNode.getTimes()[loc];
+ // skip points that do not satisfy TTL
+ if (!CommonUtils.isAlive(currTime, ttl)) {
+ results[loc] =
+ RpcUtils.getStatus(
+ TSStatusCode.OUT_OF_TTL,
+ String.format(
+ "Insertion time [%s] is less than ttl time bound [%s]",
+ DateTimeUtils.convertLongToDate(currTime),
+ DateTimeUtils.convertLongToDate(CommonDateTimeUtils.currentTime() - ttl)));
+ } else {
+ if (firstAliveLoc == -1) {
+ firstAliveLoc = loc;
+ }
+ if (breakOnFirstAlive) {
+ break;
+ }
+ }
+ loc++;
+ }
+
+ if (firstAliveLoc == -1) {
+ // no alive data
+ throw new OutOfTTLException(
+ insertTabletNode.getTimes()[insertTabletNode.getTimes().length - 1],
+ (CommonDateTimeUtils.currentTime() - ttl));
+ }
+ return firstAliveLoc;
}
/**
@@ -1078,7 +1133,8 @@ public class DataRegion implements IDataRegionForQuery {
int end,
boolean sequence,
TSStatus[] results,
- long timePartitionId) {
+ long timePartitionId,
+ boolean noFailure) {
// return when start >= end or all measurement failed
if (start >= end || insertTabletNode.allMeasurementFailed()) {
if (logger.isDebugEnabled()) {
@@ -1101,8 +1157,11 @@ public class DataRegion implements IDataRegionForQuery {
return false;
}
+ // register TableSchema (and maybe more) for table insertion
+ registerToTsFile(insertTabletNode, tsFileProcessor);
+
try {
- tsFileProcessor.insertTablet(insertTabletNode, start, end, results);
+ tsFileProcessor.insertTablet(insertTabletNode, start, end, results, noFailure);
} catch (WriteProcessRejectException e) {
logger.warn("insert to TsFileProcessor rejected, {}", e.getMessage());
return false;
@@ -1119,6 +1178,17 @@ public class DataRegion implements IDataRegionForQuery {
return true;
}
+ private void registerToTsFile(InsertNode node, TsFileProcessor tsFileProcessor) {
+ final String tableName = node.getTableName();
+ if (tableName != null) {
+ tsFileProcessor.registerToTsFile(
+ tableName,
+ t ->
+ TableSchema.of(DataNodeTableCache.getInstance().getTable(getDatabaseName(), t))
+ .toTsFileTableSchema());
+ }
+ }
+
private void tryToUpdateInsertTabletLastCache(InsertTabletNode node) {
long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID());
String[] measurements = node.getMeasurements();
@@ -1140,7 +1210,10 @@ public class DataRegion implements IDataRegionForQuery {
node.getMeasurementSchemas(),
node.isAligned(),
node::composeLastTimeValuePair,
- index -> node.getColumns()[index] != null,
+ index ->
+ node.getColumns()[index] != null
+ && (node.getColumnCategories() == null
+ || node.getColumnCategories()[index] == TsTableColumnCategory.MEASUREMENT),
true,
latestFlushedTime);
}
@@ -1182,7 +1255,10 @@ public class DataRegion implements IDataRegionForQuery {
node.getMeasurementSchemas(),
node.isAligned(),
node::composeTimeValuePair,
- index -> node.getValues()[index] != null,
+ index ->
+ node.getValues()[index] != null
+ && (node.getColumnCategories() == null
+ || node.getColumnCategories()[index] == TsTableColumnCategory.MEASUREMENT),
true,
latestFlushedTime);
}
@@ -2037,10 +2113,6 @@ public class DataRegion implements IDataRegionForQuery {
/**
* @param pattern Must be a pattern start with a precise device path
- * @param startTime
- * @param endTime
- * @param searchIndex
- * @throws IOException
*/
public void deleteByDevice(PartialPath pattern, long startTime, long endTime, long searchIndex)
throws IOException {
@@ -3253,7 +3325,7 @@ public class DataRegion implements IDataRegionForQuery {
Map<TsFileProcessor, InsertRowsNode> tsFileProcessorMap = new HashMap<>();
for (int i = 0; i < insertRowsOfOneDeviceNode.getInsertRowNodeList().size(); i++) {
InsertRowNode insertRowNode = insertRowsOfOneDeviceNode.getInsertRowNodeList().get(i);
- if (!isAlive(insertRowNode.getTime(), deviceTTL)) {
+ if (!CommonUtils.isAlive(insertRowNode.getTime(), deviceTTL)) {
// we do not need to write these part of data, as they can not be queried
// or the sub-plan has already been executed, we are retrying other sub-plans
insertRowsOfOneDeviceNode
@@ -3369,8 +3441,12 @@ public class DataRegion implements IDataRegionForQuery {
for (int i = 0; i < insertRowsNode.getInsertRowNodeList().size(); i++) {
InsertRowNode insertRowNode = insertRowsNode.getInsertRowNodeList().get(i);
long deviceTTL =
- DataNodeTTLCache.getInstance().getTTL(insertRowNode.getDevicePath().getNodes());
- if (!isAlive(insertRowNode.getTime(), deviceTTL)) {
+ DataNodeTTLCache.getInstance()
+ .getTTL(
+ Arrays.stream(insertRowNode.getDeviceID().getSegments())
+ .map(seg -> seg != null ? seg.toString() : null)
+ .toArray(String[]::new));
+ if (!CommonUtils.isAlive(insertRowNode.getTime(), deviceTTL)) {
insertRowsNode
.getResults()
.put(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index c723d1e4aca..22b1dadb394 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -19,12 +19,14 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.AlignedFullPath;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.NonAlignedFullPath;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
@@ -81,6 +83,7 @@ import java.util.stream.LongStream;
import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
public abstract class AbstractMemTable implements IMemTable {
+
/** Each memTable node has a unique int value identifier, init when recovering wal. */
public static final AtomicLong memTableIdCounter = new AtomicLong(-1);
@@ -270,7 +273,10 @@ public abstract class AbstractMemTable implements IMemTable {
List<TSDataType> dataTypes = new ArrayList<>();
for (int i = 0; i < insertRowNode.getMeasurements().length; i++) {
// Use measurements[i] to ignore failed partial insert
- if (measurements[i] == null || values[i] == null) {
+ if (measurements[i] == null
+ || values[i] == null
+ || insertRowNode.getColumnCategories() != null
+ && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.MEASUREMENT) {
schemaList.add(null);
continue;
}
@@ -281,10 +287,11 @@ public abstract class AbstractMemTable implements IMemTable {
if (schemaList.isEmpty()) {
return;
}
- memSize += MemUtils.getAlignedRowRecordSize(dataTypes, values);
+ memSize +=
+ MemUtils.getAlignedRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories());
writeAlignedRow(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values);
int pointsInserted =
- insertRowNode.getMeasurements().length - insertRowNode.getFailedMeasurementNumber();
+ insertRowNode.getMeasurementColumnCnt() - insertRowNode.getFailedMeasurementNumber();
totalPointsNum += pointsInserted;
MetricService.getInstance()
@@ -361,13 +368,16 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public void insertAlignedTablet(InsertTabletNode insertTabletNode, int start, int end)
+ public void insertAlignedTablet(
+ InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results)
throws WriteProcessException {
try {
- writeAlignedTablet(insertTabletNode, start, end);
- memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end);
+ writeAlignedTablet(insertTabletNode, start, end, results);
+ // TODO-Table: what is the relation between this and TsFileProcessor.checkMemCost
+ memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end, results);
int pointsInserted =
- (insertTabletNode.getDataTypes().length - insertTabletNode.getFailedMeasurementNumber())
+ (insertTabletNode.getMeasurementColumnCnt()
+ - insertTabletNode.getFailedMeasurementNumber())
* (end - start);
totalPointsNum += pointsInserted;
MetricService.getInstance()
@@ -446,16 +456,20 @@ public abstract class AbstractMemTable implements IMemTable {
insertTabletNode.getBitMaps(),
schemaList,
start,
- end)) {
+ end,
+ null)) {
shouldFlush = true;
}
}
- public void writeAlignedTablet(InsertTabletNode insertTabletNode, int start, int end) {
+ public void writeAlignedTablet(
+ InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) {
List<IMeasurementSchema> schemaList = new ArrayList<>();
for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) {
- if (insertTabletNode.getColumns()[i] == null) {
+ if (insertTabletNode.getColumns()[i] == null
+ || (insertTabletNode.getColumnCategories() != null
+ && insertTabletNode.getColumnCategories()[i] != TsTableColumnCategory.MEASUREMENT)) {
schemaList.add(null);
} else {
schemaList.add(insertTabletNode.getMeasurementSchemas()[i]);
@@ -464,21 +478,30 @@ public abstract class AbstractMemTable implements IMemTable {
if (schemaList.isEmpty()) {
return;
}
- IWritableMemChunkGroup memChunkGroup =
- createAlignedMemChunkGroupIfNotExistAndGet(insertTabletNode.getDeviceID(), schemaList);
- if (memChunkGroup.writeValuesWithFlushCheck(
- insertTabletNode.getTimes(),
- insertTabletNode.getColumns(),
- insertTabletNode.getBitMaps(),
- schemaList,
- start,
- end)) {
- shouldFlush = true;
+ final List<Pair<IDeviceID, Integer>> deviceEndOffsetPair =
+ insertTabletNode.splitByDevice(start, end);
+ int splitStart = start;
+ for (Pair<IDeviceID, Integer> pair : deviceEndOffsetPair) {
+ final IDeviceID deviceID = pair.left;
+ int splitEnd = pair.right;
+ IWritableMemChunkGroup memChunkGroup =
+ createAlignedMemChunkGroupIfNotExistAndGet(deviceID, schemaList);
+ if (memChunkGroup.writeValuesWithFlushCheck(
+ insertTabletNode.getTimes(),
+ insertTabletNode.getColumns(),
+ insertTabletNode.getBitMaps(),
+ schemaList,
+ splitStart,
+ splitEnd,
+ results)) {
+ shouldFlush = true;
+ }
+ splitStart = splitEnd;
}
}
@Override
- public boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement) {
+ public boolean chunkNotExist(IDeviceID deviceId, String measurement) {
IWritableMemChunkGroup memChunkGroup = memTableMap.get(deviceId);
if (null == memChunkGroup) {
return true;
@@ -1086,6 +1109,7 @@ public abstract class AbstractMemTable implements IMemTable {
}
public static class Factory {
+
private Factory() {
// Empty constructor
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index b93474fdb92..4b25096245a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils;
@@ -156,8 +157,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
@Override
public boolean putAlignedValuesWithFlushCheck(
- long[] t, Object[] v, BitMap[] bitMaps, int start, int end) {
- list.putAlignedValues(t, v, bitMaps, start, end);
+ long[] t, Object[] v, BitMap[] bitMaps, int start, int end, TSStatus[] results) {
+ list.putAlignedValues(t, v, bitMaps, start, end, results);
return list.reachMaxChunkSizeThreshold();
}
@@ -187,13 +188,14 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end) {
+ int end,
+ TSStatus[] results) {
Pair<Object[], BitMap[]> pair =
checkAndReorderColumnValuesInInsertPlan(schemaList, valueList, bitMaps);
Object[] reorderedColumnValues = pair.left;
BitMap[] reorderedBitMaps = pair.right;
return putAlignedValuesWithFlushCheck(
- times, reorderedColumnValues, reorderedBitMaps, start, end);
+ times, reorderedColumnValues, reorderedBitMaps, start, end, results);
}
/**
@@ -245,7 +247,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
return (long) list.rowCount() * measurementIndexMap.size();
}
- public long alignedListSize() {
+ public int alignedListSize() {
return list.rowCount();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
index c541e52e817..6eae996d4c5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternUtil;
@@ -55,9 +56,10 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end) {
+ int end,
+ TSStatus[] results) {
return memChunk.writeAlignedValuesWithFlushCheck(
- times, columns, bitMaps, schemaList, start, end);
+ times, columns, bitMaps, schemaList, start, end, results);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index f998a76cf1a..4de1520a22b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.PartialPath;
@@ -115,7 +116,8 @@ public interface IMemTable extends WALEntryValue {
void insertTablet(InsertTabletNode insertTabletNode, int start, int end)
throws WriteProcessException;
- void insertAlignedTablet(InsertTabletNode insertTabletNode, int start, int end)
+ void insertAlignedTablet(
+ InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results)
throws WriteProcessException;
ReadOnlyMemChunk query(
@@ -175,7 +177,7 @@ public interface IMemTable extends WALEntryValue {
void release();
/** must guarantee the device exists in the work memtable only used when mem control enabled */
- boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement);
+ boolean chunkNotExist(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
long getCurrentTVListSize(IDeviceID deviceId, String measurement);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
index eac06484035..e1070d4f06f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -58,7 +59,7 @@ public interface IWritableMemChunk extends WALEntryValue {
void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end);
boolean putAlignedValuesWithFlushCheck(
- long[] t, Object[] v, BitMap[] bitMaps, int start, int end);
+ long[] t, Object[] v, BitMap[] bitMaps, int start, int end, TSStatus[] results);
boolean writeWithFlushCheck(long insertTime, Object objectValue);
@@ -78,7 +79,8 @@ public interface IWritableMemChunk extends WALEntryValue {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end);
+ int end,
+ TSStatus[] results);
long count();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
index 3f94f361a35..883779c2344 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
@@ -36,7 +37,8 @@ public interface IWritableMemChunkGroup extends WALEntryValue {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end);
+ int end,
+ TSStatus[] results);
void release();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 769f5e6fb76..309ce4060d3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.TestOnly;
@@ -85,6 +86,7 @@ import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.writer.RestorableTsFileIOWriter;
@@ -107,6 +109,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.GET_QUERY_RESOURCE_FROM_MEM;
import static org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet.FLUSHING_MEMTABLE;
@@ -118,6 +121,8 @@ public class TsFileProcessor {
/** Logger fot this class. */
private static final Logger logger = LoggerFactory.getLogger(TsFileProcessor.class);
+ private static final int NUM_MEM_TO_ESTIMATE = 3;
+
/** Storage group name of this tsfile. */
private final String storageGroupName;
@@ -252,14 +257,7 @@ public class TsFileProcessor {
logger.info("reopen a tsfile processor {}", tsFileResource.getTsFile());
}
- /**
- * Insert data in an InsertRowNode into the workingMemtable.
- *
- * @param insertRowNode physical plan of insertion
- */
- public void insert(InsertRowNode insertRowNode, long[] costsForMetrics)
- throws WriteProcessException {
-
+ private void ensureMemTable(long[] costsForMetrics) {
if (workMemTable == null) {
long startTime = System.nanoTime();
createNewWorkingMemTable();
@@ -268,15 +266,28 @@ public class TsFileProcessor {
WritingMetrics.getInstance()
.recordActiveMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
}
+ }
+ /**
+ * Insert data in an InsertRowNode into the workingMemtable.
+ *
+ * @param insertRowNode physical plan of insertion
+ */
+ public void insert(InsertRowNode insertRowNode, long[] costsForMetrics)
+ throws WriteProcessException {
+
+ ensureMemTable(costsForMetrics);
long[] memIncrements;
long memControlStartTime = System.nanoTime();
if (insertRowNode.isAligned()) {
memIncrements =
checkAlignedMemCostAndAddToTspInfoForRow(
- insertRowNode.getDeviceID(), insertRowNode.getMeasurements(),
- insertRowNode.getDataTypes(), insertRowNode.getValues());
+ insertRowNode.getDeviceID(),
+ insertRowNode.getMeasurements(),
+ insertRowNode.getDataTypes(),
+ insertRowNode.getValues(),
+ insertRowNode.getColumnCategories());
} else {
memIncrements =
checkMemCostAndAddToTspInfoForRow(
@@ -340,14 +351,7 @@ public class TsFileProcessor {
public void insert(InsertRowsNode insertRowsNode, long[] costsForMetrics)
throws WriteProcessException {
- if (workMemTable == null) {
- long startTime = System.nanoTime();
- createNewWorkingMemTable();
- // recordCreateMemtableBlockCost
- costsForMetrics[0] += System.nanoTime() - startTime;
- WritingMetrics.getInstance()
- .recordActiveMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
- }
+ ensureMemTable(costsForMetrics);
long[] memIncrements;
@@ -421,6 +425,71 @@ public class TsFileProcessor {
walNode.onMemTableCreated(workMemTable, tsFileResource.getTsFilePath());
}
+ private long[] checkMemCost(
+ InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results, boolean noFailure)
+ throws WriteProcessException {
+ long[] memIncrements;
+ try {
+ long startTime = System.nanoTime();
+ memIncrements = checkMemCost(insertTabletNode, start, end, noFailure, results);
+ PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(System.nanoTime() - startTime);
+ } catch (WriteProcessException e) {
+ for (int i = start; i < end; i++) {
+ results[i] = RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, e.getMessage());
+ }
+ throw new WriteProcessException(e);
+ }
+
+ return memIncrements;
+ }
+
+ private long[] checkMemCost(
+ InsertTabletNode insertTabletNode, int start, int end, boolean noFailure, TSStatus[] results)
+ throws WriteProcessException {
+ long[] memIncrements;
+ if (insertTabletNode.isAligned()) {
+ memIncrements = checkAlignedMemCost(insertTabletNode, start, end, noFailure, results);
+ } else {
+ memIncrements =
+ checkMemCostAndAddToTspInfoForTablet(
+ insertTabletNode.getDeviceID(),
+ insertTabletNode.getMeasurements(),
+ insertTabletNode.getDataTypes(),
+ insertTabletNode.getColumns(),
+ start,
+ end);
+ }
+ return memIncrements;
+ }
+
+ private long[] checkAlignedMemCost(
+ InsertTabletNode insertTabletNode, int start, int end, boolean noFailure, TSStatus[] results)
+ throws WriteProcessException {
+ List<Pair<IDeviceID, Integer>> deviceEndPosList = insertTabletNode.splitByDevice(start, end);
+ long[] memIncrements = new long[NUM_MEM_TO_ESTIMATE];
+ int splitStart = start;
+ for (Pair<IDeviceID, Integer> iDeviceIDIntegerPair : deviceEndPosList) {
+ int splitEnd = iDeviceIDIntegerPair.getRight();
+ IDeviceID deviceID = iDeviceIDIntegerPair.getLeft();
+ long[] splitMemIncrements =
+ checkAlignedMemCostAndAddToTspForTablet(
+ deviceID,
+ insertTabletNode.getMeasurements(),
+ insertTabletNode.getDataTypes(),
+ insertTabletNode.getColumns(),
+ insertTabletNode.getColumnCategories(),
+ splitStart,
+ splitEnd,
+ noFailure,
+ results);
+ for (int i = 0; i < NUM_MEM_TO_ESTIMATE; i++) {
+ memIncrements[i] += splitMemIncrements[i];
+ }
+ splitStart = splitEnd;
+ }
+ return memIncrements;
+ }
+
/**
* Insert batch data of insertTabletPlan into the workingMemtable. The rows to be inserted are in
* the range [start, end). Null value in each column values will be replaced by the subsequent
@@ -432,7 +501,7 @@ public class TsFileProcessor {
* @param results result array
*/
public void insertTablet(
- InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results)
+ InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results, boolean noFailure)
throws WriteProcessException {
if (workMemTable == null) {
@@ -443,35 +512,7 @@ public class TsFileProcessor {
.recordActiveMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
}
- long[] memIncrements;
- try {
- long startTime = System.nanoTime();
- if (insertTabletNode.isAligned()) {
- memIncrements =
- checkAlignedMemCostAndAddToTspForTablet(
- insertTabletNode.getDeviceID(),
- insertTabletNode.getMeasurements(),
- insertTabletNode.getDataTypes(),
- insertTabletNode.getColumns(),
- start,
- end);
- } else {
- memIncrements =
- checkMemCostAndAddToTspInfoForTablet(
- insertTabletNode.getDeviceID(),
- insertTabletNode.getMeasurements(),
- insertTabletNode.getDataTypes(),
- insertTabletNode.getColumns(),
- start,
- end);
- }
- PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(System.nanoTime() - startTime);
- } catch (WriteProcessException e) {
- for (int i = start; i < end; i++) {
- results[i] = RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, e.getMessage());
- }
- throw new WriteProcessException(e);
- }
+ long[] memIncrements = checkMemCost(insertTabletNode, start, end, results, noFailure);
long startTime = System.nanoTime();
WALFlushListener walFlushListener;
@@ -505,7 +546,7 @@ public class TsFileProcessor {
try {
if (insertTabletNode.isAligned()) {
- workMemTable.insertAlignedTablet(insertTabletNode, start, end);
+ workMemTable.insertAlignedTablet(insertTabletNode, start, end, noFailure ? null : results);
} else {
workMemTable.insertTablet(insertTabletNode, start, end);
}
@@ -519,13 +560,24 @@ public class TsFileProcessor {
results[i] = RpcUtils.SUCCESS_STATUS;
}
+ final List<Pair<IDeviceID, Integer>> deviceEndOffsetPairs =
+ insertTabletNode.splitByDevice(start, end);
tsFileResource.updateStartTime(
- insertTabletNode.getDeviceID(), insertTabletNode.getTimes()[start]);
- // For sequence tsfile, we update the endTime only when the file is prepared to be closed.
- // For unsequence tsfile, we have to update the endTime for each insertion.
+ deviceEndOffsetPairs.get(0).left, insertTabletNode.getTimes()[start]);
if (!sequence) {
+ // For sequence tsfile, we update the endTime only when the file is prepared to be closed.
+ // For unsequence tsfile, we have to update the endTime for each insertion.
tsFileResource.updateEndTime(
- insertTabletNode.getDeviceID(), insertTabletNode.getTimes()[end - 1]);
+ deviceEndOffsetPairs.get(0).left, deviceEndOffsetPairs.get(0).right);
+ }
+ for (int i = 1; i < deviceEndOffsetPairs.size(); i++) {
+ // the end offset of i - 1 is the start offset of i
+ tsFileResource.updateStartTime(
+ deviceEndOffsetPairs.get(i).left, deviceEndOffsetPairs.get(i - 1).right);
+ if (!sequence) {
+ tsFileResource.updateEndTime(
+ deviceEndOffsetPairs.get(i).left, deviceEndOffsetPairs.get(i).right);
+ }
}
tsFileResource.updateProgressIndex(insertTabletNode.getProgressIndex());
@@ -547,7 +599,7 @@ public class TsFileProcessor {
if (dataTypes[i] == null || measurements[i] == null) {
continue;
}
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, measurements[i])) {
+ if (workMemTable.chunkNotExist(deviceId, measurements[i])) {
// ChunkMetadataIncrement
chunkMetadataIncrement += ChunkMetadata.calculateRamSize(measurements[i], dataTypes[i]);
memTableIncrement += TVList.tvListArrayMemCost(dataTypes[i]);
@@ -587,7 +639,7 @@ public class TsFileProcessor {
if (dataTypes[i] == null || measurements[i] == null) {
continue;
}
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, measurements[i])
+ if (workMemTable.chunkNotExist(deviceId, measurements[i])
&& (!increasingMemTableInfo.containsKey(deviceId)
|| !increasingMemTableInfo.get(deviceId).containsKey(measurements[i]))) {
// ChunkMetadataIncrement
@@ -621,30 +673,24 @@ public class TsFileProcessor {
@SuppressWarnings("squid:S3776") // high Cognitive Complexity
private long[] checkAlignedMemCostAndAddToTspInfoForRow(
- IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, Object[] values)
+ IDeviceID deviceId,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ Object[] values,
+ TsTableColumnCategory[] columnCategories)
throws WriteProcessException {
// Memory of increased PrimitiveArray and TEXT values, e.g., add a long[128], add 128*8
long memTableIncrement = 0L;
long textDataIncrement = 0L;
long chunkMetadataIncrement = 0L;
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)) {
+ if (workMemTable.chunkNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)) {
// For new device of this mem table
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* dataTypes.length;
- memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes);
- for (int i = 0; i < dataTypes.length; i++) {
- // Skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
- continue;
- }
- // TEXT data mem size
- if (dataTypes[i].isBinary() && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
- }
- }
+ memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories);
} else {
// For existed device of this mem table
AlignedWritableMemChunk alignedMemChunk =
@@ -653,28 +699,37 @@ public class TsFileProcessor {
List<TSDataType> dataTypesInTVList = new ArrayList<>();
for (int i = 0; i < dataTypes.length; i++) {
// Skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
+ if (dataTypes[i] == null
+ || measurements[i] == null
+ || (columnCategories != null
+ && columnCategories[i] != TsTableColumnCategory.MEASUREMENT)) {
continue;
}
- // Extending the column of aligned mem chunk
+ // add arrays for new columns
if (!alignedMemChunk.containsMeasurement(measurements[i])) {
- memTableIncrement +=
- (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.valueListArrayMemCost(dataTypes[i]);
+ int currentArrayNum =
+ alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE
+ + (alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE > 0
+ ? 1
+ : 0);
+ memTableIncrement += currentArrayNum * AlignedTVList.valueListArrayMemCost(dataTypes[i]);
dataTypesInTVList.add(dataTypes[i]);
}
- // TEXT data mem size
- if (dataTypes[i].isBinary() && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
- }
}
- // Here currentChunkPointNum >= 1
+ // this insertion will result in a new array
if ((alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE) == 0) {
dataTypesInTVList.addAll(((AlignedTVList) alignedMemChunk.getTVList()).getTsDataTypes());
memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypesInTVList);
}
}
+
+ for (int i = 0; i < dataTypes.length; i++) {
+ // TEXT data mem size
+ if (dataTypes[i] != null && dataTypes[i].isBinary() && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
+ }
updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement);
return new long[] {memTableIncrement, textDataIncrement, chunkMetadataIncrement};
}
@@ -693,27 +748,26 @@ public class TsFileProcessor {
TSDataType[] dataTypes = insertRowNode.getDataTypes();
Object[] values = insertRowNode.getValues();
String[] measurements = insertRowNode.getMeasurements();
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)
+ if (workMemTable.chunkNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)
&& !increasingMemTableInfo.containsKey(deviceId)) {
// For new device of this mem table
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* dataTypes.length;
- memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes);
+ memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null);
for (int i = 0; i < dataTypes.length; i++) {
// Skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
+ if (dataTypes[i] == null
+ || measurements[i] == null
+ || (insertRowNode.getColumnCategories() != null
+ && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.MEASUREMENT)) {
continue;
}
increasingMemTableInfo
.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 1))
.left
.put(measurements[i], dataTypes[i]);
- // TEXT data mem size
- if (dataTypes[i].isBinary() && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
- }
}
} else {
@@ -722,13 +776,16 @@ public class TsFileProcessor {
(AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId);
AlignedWritableMemChunk alignedMemChunk =
memChunkGroup == null ? null : memChunkGroup.getAlignedMemChunk();
- long currentChunkPointNum = alignedMemChunk == null ? 0 : alignedMemChunk.alignedListSize();
+ int currentChunkPointNum = alignedMemChunk == null ? 0 : alignedMemChunk.alignedListSize();
List<TSDataType> dataTypesInTVList = new ArrayList<>();
Pair<Map<String, TSDataType>, Integer> addingPointNumInfo =
increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0));
for (int i = 0; i < dataTypes.length; i++) {
// Skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
+ if (dataTypes[i] == null
+ || measurements[i] == null
+ || (insertRowNode.getColumnCategories() != null
+ && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.MEASUREMENT)) {
continue;
}
@@ -736,14 +793,15 @@ public class TsFileProcessor {
// Extending the column of aligned mem chunk
if ((alignedMemChunk != null && !alignedMemChunk.containsMeasurement(measurements[i]))
&& !increasingMemTableInfo.get(deviceId).left.containsKey(measurements[i])) {
- memTableIncrement +=
- ((currentChunkPointNum + addingPointNum) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.valueListArrayMemCost(dataTypes[i]);
increasingMemTableInfo.get(deviceId).left.put(measurements[i], dataTypes[i]);
- }
- // TEXT data mem size
- if (dataTypes[i].isBinary() && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ int currentArrayNum =
+ (currentChunkPointNum + addingPointNum) / PrimitiveArrayManager.ARRAY_SIZE
+ + ((currentChunkPointNum + addingPointNum) % PrimitiveArrayManager.ARRAY_SIZE
+ > 0
+ ? 1
+ : 0);
+ memTableIncrement +=
+ currentArrayNum * AlignedTVList.valueListArrayMemCost(dataTypes[i]);
}
}
int addingPointNum = increasingMemTableInfo.get(deviceId).getRight();
@@ -758,6 +816,20 @@ public class TsFileProcessor {
}
increasingMemTableInfo.get(deviceId).setRight(addingPointNum + 1);
}
+
+ for (int i = 0; i < dataTypes.length; i++) {
+ // Skip failed Measurements
+ if (dataTypes[i] == null
+ || measurements[i] == null
+ || (insertRowNode.getColumnCategories() != null
+ && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.MEASUREMENT)) {
+ continue;
+ }
+ // TEXT data mem size
+ if (dataTypes[i].isBinary() && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
+ }
}
updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement);
return new long[] {memTableIncrement, textDataIncrement, chunkMetadataIncrement};
@@ -795,15 +867,28 @@ public class TsFileProcessor {
String[] measurements,
TSDataType[] dataTypes,
Object[] columns,
+ TsTableColumnCategory[] columnCategories,
int start,
- int end)
+ int end,
+ boolean noFailure,
+ TSStatus[] results)
throws WriteProcessException {
if (start >= end) {
return new long[] {0, 0, 0};
}
long[] memIncrements = new long[3]; // memTable, text, chunk metadata
- updateAlignedMemCost(dataTypes, deviceId, measurements, start, end, memIncrements, columns);
+ updateAlignedMemCost(
+ dataTypes,
+ deviceId,
+ measurements,
+ start,
+ end,
+ memIncrements,
+ columns,
+ columnCategories,
+ noFailure,
+ results);
long memTableIncrement = memIncrements[0];
long textDataIncrement = memIncrements[1];
long chunkMetadataIncrement = memIncrements[2];
@@ -821,7 +906,7 @@ public class TsFileProcessor {
Object column) {
// memIncrements = [memTable, text, chunk metadata] respectively
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, measurement)) {
+ if (workMemTable.chunkNotExist(deviceId, measurement)) {
// ChunkMetadataIncrement
memIncrements[2] += ChunkMetadata.calculateRamSize(measurement, dataType);
memIncrements[0] +=
@@ -845,7 +930,7 @@ public class TsFileProcessor {
// TEXT data size
if (dataType.isBinary()) {
Binary[] binColumn = (Binary[]) column;
- memIncrements[1] += MemUtils.getBinaryColumnSize(binColumn, start, end);
+ memIncrements[1] += MemUtils.getBinaryColumnSize(binColumn, start, end, null);
}
}
@@ -856,72 +941,110 @@ public class TsFileProcessor {
int start,
int end,
long[] memIncrements,
- Object[] columns) {
- // memIncrements = [memTable, text, chunk metadata] respectively
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)) {
- // ChunkMetadataIncrement
- memIncrements[2] +=
- dataTypes.length
- * ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR);
- memIncrements[0] +=
- ((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
- for (int i = 0; i < dataTypes.length; i++) {
- TSDataType dataType = dataTypes[i];
- String measurement = measurementIds[i];
- Object column = columns[i];
- if (dataType == null || column == null || measurement == null) {
- continue;
+ Object[] columns,
+ TsTableColumnCategory[] columnCategories,
+ boolean noFailure,
+ TSStatus[] results) {
+ int incomingPointNum;
+ if (noFailure) {
+ incomingPointNum = end - start;
+ } else {
+ incomingPointNum = end - start;
+ for (TSStatus result : results) {
+ if (result != null && result.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ incomingPointNum--;
}
- // TEXT data size
- if (dataType.isBinary()) {
- Binary[] binColumn = (Binary[]) columns[i];
- memIncrements[1] += MemUtils.getBinaryColumnSize(binColumn, start, end);
+ }
+ }
+
+ int measurementColumnNum = 0;
+ if (columnCategories == null) {
+ measurementColumnNum = dataTypes.length;
+ } else {
+ for (TsTableColumnCategory columnCategory : columnCategories) {
+ if (columnCategory == TsTableColumnCategory.MEASUREMENT) {
+ measurementColumnNum++;
}
}
+ }
+ // memIncrements = [memTable, text, chunk metadata] respectively
+ if (workMemTable.chunkNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)) {
+ // new devices introduce new ChunkMetadata
+ // ChunkMetadata memory Increment
+ memIncrements[2] +=
+ measurementColumnNum
+ * ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR);
+ // TVList memory
+
+ int numArraysToAdd =
+ incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE
+ + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0);
+ memIncrements[0] +=
+ numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories);
} else {
AlignedWritableMemChunk alignedMemChunk =
((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
.getAlignedMemChunk();
List<TSDataType> dataTypesInTVList = new ArrayList<>();
+ int currentPointNum = alignedMemChunk.alignedListSize();
+ int newPointNum = currentPointNum + incomingPointNum;
for (int i = 0; i < dataTypes.length; i++) {
TSDataType dataType = dataTypes[i];
String measurement = measurementIds[i];
Object column = columns[i];
- if (dataType == null || column == null || measurement == null) {
+ if (dataType == null
+ || column == null
+ || measurement == null
+ || (columnCategories != null
+ && columnCategories[i] != TsTableColumnCategory.MEASUREMENT)) {
continue;
}
- // Extending the column of aligned mem chunk
+
if (!alignedMemChunk.containsMeasurement(measurementIds[i])) {
+ // add a new column in the TVList, the new column should be as long as existing ones
memIncrements[0] +=
- (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
+ (currentPointNum / PrimitiveArrayManager.ARRAY_SIZE + 1)
* AlignedTVList.valueListArrayMemCost(dataType);
dataTypesInTVList.add(dataType);
}
- // TEXT data size
- if (dataType.isBinary()) {
- Binary[] binColumn = (Binary[]) columns[i];
- memIncrements[1] += MemUtils.getBinaryColumnSize(binColumn, start, end);
- }
- }
- long acquireArray;
- if (alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE == 0) {
- acquireArray = (end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1L;
- } else {
- acquireArray =
- (end
- - start
- - 1
- + (alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE))
- / PrimitiveArrayManager.ARRAY_SIZE;
}
+
+ // calculate how many new arrays will be added after this insertion
+ int currentArrayCnt =
+ currentPointNum / PrimitiveArrayManager.ARRAY_SIZE
+ + (currentPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0);
+ int newArrayCnt =
+ newPointNum / PrimitiveArrayManager.ARRAY_SIZE
+ + (newPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0);
+ long acquireArray = newArrayCnt - currentArrayCnt;
+
if (acquireArray != 0) {
+ // memory of extending the TVList
dataTypesInTVList.addAll(((AlignedTVList) alignedMemChunk.getTVList()).getTsDataTypes());
memIncrements[0] +=
acquireArray * AlignedTVList.alignedTvListArrayMemCost(dataTypesInTVList);
}
}
+
+ // flexible-length data size
+ for (int i = 0; i < dataTypes.length; i++) {
+ TSDataType dataType = dataTypes[i];
+ String measurement = measurementIds[i];
+ Object column = columns[i];
+ if (dataType == null
+ || column == null
+ || measurement == null
+ || (columnCategories != null
+ && columnCategories[i] != TsTableColumnCategory.MEASUREMENT)) {
+ continue;
+ }
+
+ if (dataType.isBinary()) {
+ Binary[] binColumn = (Binary[]) columns[i];
+ memIncrements[1] += MemUtils.getBinaryColumnSize(binColumn, start, end, results);
+ }
+ }
}
private void updateMemoryInfo(
@@ -2143,4 +2266,12 @@ public class TsFileProcessor {
public ConcurrentLinkedDeque<IMemTable> getFlushingMemTable() {
return flushingMemTables;
}
+
+ public void registerToTsFile(
+ String tableName, Function<String, TableSchema> tableSchemaFunction) {
+ getWriter()
+ .getKnownSchema()
+ .getTableSchemaMap()
+ .computeIfAbsent(tableName, tableSchemaFunction);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index 4537809f5af..7a753658c96 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
@@ -137,7 +138,8 @@ public class WritableMemChunk implements IWritableMemChunk {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end) {
+ int end,
+ TSStatus[] results) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + list.getDataType());
}
@@ -211,7 +213,7 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
public boolean putAlignedValuesWithFlushCheck(
- long[] t, Object[] v, BitMap[] bitMaps, int start, int end) {
+ long[] t, Object[] v, BitMap[] bitMaps, int start, int end, TSStatus[] results) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
index 5d61fe0e3dd..b8a868cecb0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternUtil;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
@@ -51,7 +52,8 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
BitMap[] bitMaps,
List<IMeasurementSchema> schemaList,
int start,
- int end) {
+ int end,
+ TSStatus[] results) {
boolean flushFlag = false;
for (int i = 0; i < columns.length; i++) {
if (columns[i] == null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java
index 162b1ac02b2..1340554af99 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java
@@ -100,7 +100,7 @@ public class TsFilePlanRedoer {
} else {
if (node.isAligned()) {
recoveryMemTable.insertAlignedTablet(
- (InsertTabletNode) node, 0, ((InsertTabletNode) node).getRowCount());
+ (InsertTabletNode) node, 0, ((InsertTabletNode) node).getRowCount(), null);
} else {
recoveryMemTable.insertTablet(
(InsertTabletNode) node, 0, ((InsertTabletNode) node).getRowCount());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java
index 9fc5e21b7a4..80f41592387 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java
@@ -43,6 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.trigger.service.TriggerManagementService;
import org.apache.iotdb.mpp.rpc.thrift.TFireTriggerReq;
import org.apache.iotdb.mpp.rpc.thrift.TFireTriggerResp;
@@ -133,6 +134,13 @@ public class TriggerFireVisitor extends PlanVisitor<TriggerFireResult, TriggerEv
return hasFailedTrigger ? TriggerFireResult.FAILED_NO_TERMINATION : TriggerFireResult.SUCCESS;
}
+ @Override
+ public TriggerFireResult visitRelationalInsertTablet(
+ RelationalInsertTabletNode node, TriggerEvent context) {
+ // TODO-Table: add support
+ return visitInsertTablet(node, context);
+ }
+
@Override
public TriggerFireResult visitInsertTablet(InsertTabletNode node, TriggerEvent context) {
// group Triggers and measurements
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
index 9b902d0cf44..59cc90ab219 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.utils;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
@@ -43,8 +45,11 @@ import io.airlift.airline.ParseOptionMissingValueException;
import org.apache.commons.lang3.StringUtils;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.UnSupportedDataTypeException;
+import java.time.LocalDate;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.List;
@@ -397,4 +402,76 @@ public class CommonUtils {
System.err.println("-- StackTrace --");
System.err.println(Throwables.getStackTraceAsString(e));
}
+
+ public static String[] deviceIdToStringArray(IDeviceID deviceID) {
+ String[] ret = new String[deviceID.segmentNum()];
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = deviceID.segment(i) != null ? deviceID.segment(i).toString() : null;
+ }
+ return ret;
+ }
+
+ public static Object[] deviceIdToObjArray(IDeviceID deviceID) {
+ Object[] ret = new Object[deviceID.segmentNum()];
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = deviceID.segment(i);
+ }
+ return ret;
+ }
+
+ /**
+ * Check whether the time falls in TTL.
+ *
+ * @return whether the given time falls in ttl
+ */
+ public static boolean isAlive(long time, long dataTTL) {
+ return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL;
+ }
+
+ public static Object createValueColumnOfDataType(
+ TSDataType dataType, TsTableColumnCategory columnCategory, int rowNum) {
+ Object valueColumn;
... 2667 lines suppressed ...