You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2026/10/22 05:57:38 UTC
(iotdb) branch master updated: Support auto-sync of tree-to-table views: detect new measurements and add the matching columns (#18874)
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3be5e53f214 Support auto-sync of tree-to-table views: detect new measurements and add the matching columns (#18874)
3be5e53f214 is described below
commit 3be5e53f2149e44ab87501b3d4c63480c63576f1
Author: alpass163 <al...@163.com>
AuthorDate: Thu Oct 22 13:57:31 2026 +0800
Support auto-sync of tree-to-table views: detect new measurements and add the matching columns (#18874)
---
.../iotdb/relational/it/schema/IoTDBTableIT.java | 25 +-
.../it/schema/IoTDBTreeToTableViewAutoSyncIT.java | 717 +++++++++++++++++++++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../iotdb/confignode/i18n/ManagerMessages.java | 3 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 9 +-
.../iotdb/confignode/i18n/ManagerMessages.java | 3 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 6 +
.../iotdb/confignode/manager/ConfigManager.java | 14 +-
.../apache/iotdb/confignode/manager/IManager.java | 4 +
.../iotdb/confignode/manager/ProcedureManager.java | 277 +++++++-
.../manager/schema/ClusterSchemaManager.java | 117 ++--
.../persistence/schema/ClusterSchemaInfo.java | 15 +-
.../schema/TreeDeviceViewFieldDetector.java | 305 +++++----
.../procedure/impl/schema/SchemaUtils.java | 54 +-
.../table/AbstractAlterOrDropTableProcedure.java | 34 +-
.../table/AbstractSetPropertiesProcedure.java | 2 +-
.../impl/schema/table/AddTableColumnProcedure.java | 6 +-
.../table/AlterTableColumnDataTypeProcedure.java | 2 +-
.../impl/schema/table/CreateTableProcedure.java | 44 +-
.../impl/schema/table/DropTableProcedure.java | 4 +-
.../schema/table/RenameTableColumnProcedure.java | 2 +-
.../schema/table/view/AddViewColumnProcedure.java | 55 +-
.../table/view/CreateTableViewProcedure.java | 84 ++-
.../impl/schema/table/view/DropViewProcedure.java | 15 +
.../schema/table/view/RenameViewProcedure.java | 19 +
.../table/view/SetViewPropertiesProcedure.java | 83 +++
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +
.../assembly/resources/conf/logback-datanode.xml | 21 +
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 9 +-
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 21 +
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 10 +-
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 21 +
.../schemaregion/SchemaExecutionVisitor.java | 238 ++++++-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 9 +
.../impl/DataNodeInternalRPCServiceImpl.java | 102 ++-
.../execution/config/TableConfigTaskVisitor.java | 25 +-
.../metadata/relational/ShowCreateViewTask.java | 7 +-
.../schemaengine/lease/MetadataLeaseManager.java | 4 +-
.../view/TreeToTableViewReference.java | 94 +++
.../db/schemaengine/table/DataNodeTableCache.java | 176 ++++-
.../iotdb/db/schemaengine/table/ITableCache.java | 11 +-
.../schemaengine/table/ITreeToTableViewCache.java | 70 ++
.../schemaengine/table/TreeToTableViewCache.java | 501 ++++++++++++++
.../table/ViewColumnAutoSyncTrigger.java | 337 ++++++++++
.../dataregion/IoTConsensusShutdownTest.java | 2 +-
.../SchemaRegionSimpleRecoverTest.java | 4 +-
...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 2 +-
.../plan/relational/analyzer/AnalyzerTest.java | 10 +-
.../plan/relational/analyzer/TreeViewTest.java | 3 +-
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 15 +-
.../TopKRuntimeFilterTreeViewPlanTest.java | 3 +-
.../plan/statement/InsertStatementTest.java | 6 +-
.../metric/SchemaMemMetricTableTest.java | 2 +-
.../table/DataNodeTableCacheLeaseTest.java | 4 +-
.../schemaengine/table/DataNodeTableCacheTest.java | 20 +-
.../table/TreeToTableViewCacheTest.java | 132 ++++
.../storageengine/dataregion/DataRegionTest.java | 3 +-
.../CompactionDataTypeAlterTableTest.java | 5 +-
.../TableModelCompactionWithTTLTest.java | 2 +-
.../file/UnsealedTsFileRecoverPerformerTest.java | 2 +-
.../load/LoadTsFileTableSchemaTest.java | 2 +-
.../apache/iotdb/commons/i18n/SchemaMessages.java | 2 +
.../apache/iotdb/commons/i18n/SchemaMessages.java | 2 +
.../table/TreeToTableViewTrieOperationType.java | 68 ++
.../iotdb/commons/schema/table/TreeViewSchema.java | 19 +
.../apache/iotdb/commons/schema/table/TsTable.java | 1 +
.../src/main/thrift/confignode.thrift | 19 +
.../src/main/thrift/datanode.thrift | 9 +
70 files changed, 3547 insertions(+), 357 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
index f7a5bc4581c..037ac00cd1f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
@@ -1038,24 +1038,13 @@ public class IoTDBTableIT {
final Statement statement = connection.createStatement()) {
statement.execute("use tree_view_db");
- try {
- statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**");
- fail();
- } catch (final SQLException e) {
- assertEquals(
- "614: Multiple types encountered when auto detecting type of measurement 'S1', please check",
- e.getMessage());
- }
-
- try {
- statement.execute(
- "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**");
- fail();
- } catch (final SQLException e) {
- assertEquals(
- "614: Multiple types encountered when auto detecting type of measurement 'S1', please check",
- e.getMessage());
- }
+ // Template t1 is set but not activated, so its S1 BOOLEAN is not a created series and does
+ // not take part in the detection
+ statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**");
+ statement.execute("drop view tree_table");
+ statement.execute(
+ "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**");
+ statement.execute("drop view tree_table");
}
try (final Connection connection = EnvFactory.getEnv().getConnection();
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java
new file mode 100644
index 00000000000..17dfee84836
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java
@@ -0,0 +1,717 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.schema;
+
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBTreeToTableViewAutoSyncIT {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ // ==================== the property and its index (Trie) ====================
+
+ /**
+ * The property defaults to false: when the view is created, only the series which exist become
+ * columns. An explicit false behaves the same, and SHOW CREATE VIEW only shows the property when
+ * it is stored.
+ */
+ @Test
+ @Ignore
+ public void testAutoSyncSchemaDisabledByDefault() throws Exception {
+ final String treeDb = "root.as01";
+ final String viewDb = "as01";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32");
+
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(table, "CREATE VIEW v1 (dev STRING TAG) AS " + treeDb + ".**");
+ // the creation always scans the series which already exist
+ awaitFields(table, viewDb, "v1", fields("s1", "INT32"));
+ Assert.assertFalse(showCreateView(table, viewDb, "v1").contains("auto_sync_schema"));
+
+ // the default is false, so a new measurement is not added
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s2 INT32");
+ awaitAbsent(table, viewDb, "v1", "s2");
+
+ execute(
+ table,
+ "CREATE VIEW v2 (dev STRING TAG) WITH (auto_sync_schema=false) AS " + treeDb + ".**");
+ awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32"));
+ Assert.assertTrue(showCreateView(table, viewDb, "v2").contains("auto_sync_schema=false"));
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s3 INT32");
+ awaitAbsent(table, viewDb, "v2", "s3");
+
+ // an explicit true follows the new measurements, both the existing and the later ones
+ execute(
+ table,
+ "CREATE VIEW v3 (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**");
+ awaitFields(table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32"));
+ Assert.assertTrue(showCreateView(table, viewDb, "v3").contains("auto_sync_schema=true"));
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s4 INT32");
+ awaitFields(
+ table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32"));
+ }
+ }
+
+ /** The property is only accepted by the views created on a tree path, not by the tables. */
+ @Test
+ @Ignore
+ public void testAutoSyncSchemaOnlyForTreeToTableViews() throws Exception {
+ final String viewDb = "as02";
+ try (Connection table = tableConnection()) {
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(table, "CREATE TABLE t1 (tag1 STRING TAG, s1 INT32 FIELD)");
+
+ assertSqlError(
+ table,
+ "CREATE TABLE t2 (tag1 STRING TAG, s1 INT32 FIELD) WITH (auto_sync_schema=true)",
+ "auto_sync_schema");
+ assertSqlError(
+ table, "ALTER TABLE t1 SET PROPERTIES auto_sync_schema=true", "auto_sync_schema");
+ }
+ }
+
+ /**
+ * A measurement whose name only differs in case from an existing column, or whose name is the
+ * same but whose type is different, does not change the view; the other measurements of the same
+ * write are still added.
+ */
+ @Test
+ @Ignore
+ public void testConflictingMeasurementsDoNotChangeTheView() throws Exception {
+ final String treeDb = "root.as03";
+ final String viewDb = "as03";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 DOUBLE");
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**");
+ awaitFields(table, viewDb, "v", fields("s1", "DOUBLE"));
+
+ // the same name with another type: the series is created, the column is not touched
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s1 INT32");
+ // a new measurement of the same write is still added
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32"));
+
+ // the same name in another case is a conflict as well: no column is added
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.S2 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32"));
+ }
+ }
+
+ /**
+ * A restrict view matches a device with exactly numOfTags levels under its prefix, a non-restrict
+ * view matches every device with at most numOfTags levels. A device may be covered by several
+ * views at once.
+ */
+ @Test
+ @Ignore
+ public void testTrieMatchingOfRestrictAndNonRestrictViews() throws Exception {
+ final String treeDb = "root.as04";
+ final String viewDb = "as04";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v_restrict (a STRING TAG, b STRING TAG) RESTRICT "
+ + "WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".**");
+ execute(
+ table,
+ "CREATE VIEW v_non_restrict (a STRING TAG) "
+ + "WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".**");
+
+ // one level under the prefix: only the non-restrict view matches
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32");
+ awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32"));
+ awaitAbsent(table, viewDb, "v_restrict", "s1");
+
+ // two levels: only the restrict view matches
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32");
+ awaitFields(table, viewDb, "v_restrict", fields("s2", "INT32"));
+ awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32"));
+
+ // three levels: neither view matches
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32");
+ awaitAbsent(table, viewDb, "v_restrict", "s3");
+ awaitAbsent(table, viewDb, "v_non_restrict", "s3");
+ }
+ }
+
+ /** Renaming a view keeps its place in the index; the property can be switched on and off. */
+ @Test
+ @Ignore
+ public void testTrieRenameAndToggleAutoSyncSchema() throws Exception {
+ final String treeDb = "root.as05";
+ final String viewDb = "as05";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "INT32"));
+
+ execute(table, "ALTER VIEW v RENAME TO v2");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32");
+ awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32"));
+ assertSqlError(table, "DESC v", "does not exist");
+
+ // turned off: the new measurements are ignored
+ execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=false");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.s3 INT32");
+ awaitAbsent(table, viewDb, "v2", "s3");
+
+ // turned on again: the missing columns are backfilled, and the later ones are followed
+ execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=true");
+ awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32"));
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d4.s4 INT32");
+ awaitFields(
+ table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32"));
+ }
+ }
+
+ /**
+ * CREATE OR REPLACE registers the new definition of the view: the old path stops being followed,
+ * and the new matching attributes take effect.
+ */
+ @Test
+ @Ignore
+ public void testTrieReplaceView() throws Exception {
+ final String treeDb = "root.as06";
+ final String viewDb = "as06";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "INT32"));
+
+ // another source path: the new definition replaces the old one
+ execute(
+ table,
+ "CREATE OR REPLACE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".new.**");
+ awaitFields(table, viewDb, "v", fields());
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32");
+ awaitAbsent(table, viewDb, "v", "s2");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32");
+ awaitFields(table, viewDb, "v", fields("s3", "INT32"));
+
+ // the same path with another tag count: the new matching attributes replace the old ones
+ execute(
+ table,
+ "CREATE VIEW w (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".replace.**");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.s1 INT32");
+ awaitFields(table, viewDb, "w", fields("s1", "INT32"));
+ execute(
+ table,
+ "CREATE OR REPLACE VIEW w (a STRING TAG, b STRING TAG) RESTRICT "
+ + "WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".replace.**");
+ awaitFields(table, viewDb, "w", fields());
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.q.s2 INT32");
+ awaitFields(table, viewDb, "w", fields("s2", "INT32"));
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.m.s3 INT32");
+ awaitAbsent(table, viewDb, "w", "s3");
+ }
+ }
+
+ /** Dropping a view or its database removes the entry from the index of the DataNodes. */
+ @Test
+ @Ignore
+ public void testTrieDropViewAndDropDatabase() throws Exception {
+ final String treeDb = "root.as07";
+ final String viewDb = "as07";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "INT32"));
+
+ execute(table, "DROP VIEW v");
+ assertSqlError(table, "DESC v", "does not exist");
+ // a view of another path follows only its own path
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".new.**");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32");
+ awaitAbsent(table, viewDb, "v", "s2");
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32");
+ awaitFields(table, viewDb, "v", fields("s3", "INT32"));
+ }
+
+ // dropping the database removes its views from the index: a view of the same name created
+ // again with another source path is not polluted by the old one
+ final String replacedTreeDb = "root.as07b";
+ final String replacedViewDb = "as07b";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + replacedTreeDb);
+ execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d1.s1 INT32");
+ execute(table, "CREATE DATABASE " + replacedViewDb);
+ execute(table, "USE " + replacedViewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS "
+ + replacedTreeDb
+ + ".old.**");
+ awaitFields(table, replacedViewDb, "v", fields("s1", "INT32"));
+
+ execute(table, "DROP DATABASE " + replacedViewDb);
+ execute(table, "CREATE DATABASE " + replacedViewDb);
+ execute(table, "USE " + replacedViewDb);
+ execute(
+ table,
+ "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS "
+ + replacedTreeDb
+ + ".new.**");
+
+ execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d2.s2 INT32");
+ awaitAbsent(table, replacedViewDb, "v", "s2");
+ execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".new.d3.s3 INT32");
+ awaitFields(table, replacedViewDb, "v", fields("s3", "INT32"));
+ }
+ }
+
+ /** Adding a tag column changes the tag count the view matches devices with. */
+ @Test
+ @Ignore
+ public void testTrieAddingTagColumnUpdatesTheTagCount() throws Exception {
+ final String treeDb = "root.as08";
+ final String viewDb = "as08";
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".**");
+
+ // the tag count of the view, hence the devices it matches, changes with the new tag column
+ execute(table, "ALTER VIEW v ADD COLUMN b STRING TAG");
+
+ // two levels under the prefix now match the restrict view
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32");
+ awaitFields(table, viewDb, "v", fields("s2", "INT32"));
+ // one level does not match any more
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32");
+ awaitAbsent(table, viewDb, "v", "s1");
+ // three levels do not match either
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32");
+ awaitAbsent(table, viewDb, "v", "s3");
+ }
+ }
+
+ // ==================== the interfaces which create measurements ====================
+
+ @Test
+ @Ignore
+ public void testCreateTimeSeriesAddsColumn() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as10";
+ final String viewDb = "as10";
+ prepareView(tree, table, treeDb, viewDb);
+ execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32");
+ awaitFields(table, viewDb, "v", fields("s1", "INT32"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testCreateAlignedTimeSeriesAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as11";
+ final String viewDb = "as11";
+ prepareView(tree, table, treeDb, viewDb);
+ execute(tree, "CREATE ALIGNED TIMESERIES " + treeDb + ".d1 (s1 INT32, s2 DOUBLE)");
+ awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testCreateMultiTimeseriesAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection();
+ ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ final String treeDb = "root.as12";
+ final String viewDb = "as12";
+ prepareView(tree, table, treeDb, viewDb);
+ session.createMultiTimeseries(
+ Arrays.asList(treeDb + ".d1.s1", treeDb + ".d1.s2"),
+ Arrays.asList(TSDataType.INT32, TSDataType.DOUBLE),
+ Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN),
+ Arrays.asList(CompressionType.LZ4, CompressionType.LZ4),
+ null,
+ null,
+ null,
+ null);
+ awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testInsertAutoCreatesColumn() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as13";
+ final String viewDb = "as13";
+ prepareView(tree, table, treeDb, viewDb);
+ execute(tree, "INSERT INTO " + treeDb + ".d1(time, s1) VALUES (1, 1.0)");
+ awaitFields(table, viewDb, "v", fields("s1", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testInsertMultipleDevicesAutoCreatesColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection();
+ ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ final String treeDb = "root.as14";
+ final String viewDb = "as14";
+ prepareView(tree, table, treeDb, viewDb);
+ final List<Object> valuesOfD1 = Collections.singletonList(1);
+ final List<Object> valuesOfD2 = Collections.singletonList(2.0);
+ session.insertRecords(
+ Arrays.asList(treeDb + ".d1", treeDb + ".d2"),
+ Arrays.asList(1L, 1L),
+ Arrays.asList(Collections.singletonList("s1"), Collections.singletonList("s2")),
+ Arrays.asList(
+ Collections.singletonList(TSDataType.INT32),
+ Collections.singletonList(TSDataType.DOUBLE)),
+ Arrays.asList(valuesOfD1, valuesOfD2));
+ awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testActivateTemplateAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as15";
+ final String viewDb = "as15";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as15 (s1 FLOAT, s2 DOUBLE)");
+ execute(tree, "SET DEVICE TEMPLATE t_as15 TO " + treeDb);
+ createView(table, viewDb, treeDb);
+
+ execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testBatchActivateTemplateAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection();
+ ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ final String treeDb = "root.as16";
+ final String viewDb = "as16";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as16 (s1 FLOAT, s2 DOUBLE)");
+ execute(tree, "SET DEVICE TEMPLATE t_as16 TO " + treeDb);
+ createView(table, viewDb, treeDb);
+
+ session.createTimeseriesUsingSchemaTemplate(Arrays.asList(treeDb + ".d1", treeDb + ".d2"));
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testInternalBatchActivateTemplateAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection();
+ ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ final String treeDb = "root.as17";
+ final String viewDb = "as17";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as17 (s1 FLOAT, s2 DOUBLE)");
+ execute(tree, "SET DEVICE TEMPLATE t_as17 TO " + treeDb);
+ createView(table, viewDb, treeDb);
+
+ // the insert activates the template on both devices and creates the template measurements
+ final List<Object> valuesOfD1 = Arrays.asList(1.0f, 1.0);
+ final List<Object> valuesOfD2 = Arrays.asList(2.0f, 2.0);
+ session.insertRecords(
+ Arrays.asList(treeDb + ".d1", treeDb + ".d2"),
+ Arrays.asList(1L, 1L),
+ Arrays.asList(Arrays.asList("s1", "s2"), Arrays.asList("s1", "s2")),
+ Arrays.asList(
+ Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE),
+ Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE)),
+ Arrays.asList(valuesOfD1, valuesOfD2));
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testAlterTemplateAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as18";
+ final String viewDb = "as18";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as18 (s1 FLOAT)");
+ execute(tree, "SET DEVICE TEMPLATE t_as18 TO " + treeDb);
+ createView(table, viewDb, treeDb);
+ execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT"));
+
+ execute(tree, "ALTER DEVICE TEMPLATE t_as18 ADD (s2 DOUBLE)");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ /** A device which is exactly the path the template is set on uses the template as well. */
+ @Test
+ @Ignore
+ public void testAlterTemplateOnDevicePathAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as19";
+ final String viewDb = "as19";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as19 (s1 FLOAT)");
+ execute(tree, "SET DEVICE TEMPLATE t_as19 TO " + treeDb + ".exact.d");
+ createView(table, viewDb, treeDb, "a STRING TAG, b STRING TAG, c STRING TAG");
+
+ execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".exact.d");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT"));
+
+ execute(tree, "ALTER DEVICE TEMPLATE t_as19 ADD (s2 DOUBLE)");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ /** The template may be set above the database which holds the activated devices. */
+ @Test
+ @Ignore
+ public void testAlterTemplateOnAncestorPathAddsColumns() throws Exception {
+ try (Connection tree = treeConnection();
+ Connection table = tableConnection()) {
+ final String treeDb = "root.as20.db";
+ final String viewDb = "as20";
+ execute(tree, "CREATE DATABASE " + treeDb);
+ execute(tree, "CREATE DEVICE TEMPLATE t_as20 (s1 FLOAT)");
+ execute(tree, "SET DEVICE TEMPLATE t_as20 TO root.as20");
+ createView(table, viewDb, "root.as20", "a STRING TAG, b STRING TAG");
+
+ execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT"));
+
+ execute(tree, "ALTER DEVICE TEMPLATE t_as20 ADD (s2 DOUBLE)");
+ awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE"));
+ }
+ }
+
+ // ==================== helpers ====================
+
+ private static Connection treeConnection() throws SQLException {
+ return EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
+ }
+
+ private static Connection tableConnection() throws SQLException {
+ return EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ }
+
+ private static void execute(final Connection connection, final String sql) throws SQLException {
+ try (Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ }
+ }
+
+ private static void prepareView(
+ final Connection tree, final Connection table, final String treeDb, final String viewDb)
+ throws SQLException {
+ execute(tree, "CREATE DATABASE " + treeDb);
+ createView(table, viewDb, treeDb);
+ }
+
+ private static void createView(final Connection table, final String viewDb, final String treeDb)
+ throws SQLException {
+ createView(table, viewDb, treeDb, "dev STRING TAG");
+ }
+
+ private static void createView(
+ final Connection table,
+ final String viewDb,
+ final String treeDb,
+ final String columnDefinitions)
+ throws SQLException {
+ execute(table, "CREATE DATABASE " + viewDb);
+ execute(table, "USE " + viewDb);
+ execute(
+ table,
+ "CREATE VIEW v ("
+ + columnDefinitions
+ + ") WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".**");
+ }
+
+ /** The FIELD columns of the view, column name to data type. */
+ private static Map<String, String> fieldColumns(
+ final Connection table, final String viewDb, final String view) throws SQLException {
+ execute(table, "USE " + viewDb);
+ final Map<String, String> fields = new HashMap<>();
+ try (Statement statement = table.createStatement();
+ ResultSet resultSet = statement.executeQuery("DESC " + view)) {
+ while (resultSet.next()) {
+ if ("FIELD".equalsIgnoreCase(resultSet.getString("Category"))) {
+ fields.put(resultSet.getString("ColumnName"), resultSet.getString("DataType"));
+ }
+ }
+ }
+ return fields;
+ }
+
+ private static String showCreateView(
+ final Connection table, final String viewDb, final String view) throws SQLException {
+ execute(table, "USE " + viewDb);
+ try (Statement statement = table.createStatement();
+ ResultSet resultSet = statement.executeQuery("SHOW CREATE VIEW " + view)) {
+ Assert.assertTrue(resultSet.next());
+ return resultSet.getString(2);
+ }
+ }
+
+ private static void assertSqlError(
+ final Connection connection, final String sql, final String expectedMessage) {
+ try (Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ Assert.fail("Expected an error for: " + sql);
+ } catch (final SQLException e) {
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains(expectedMessage));
+ }
+ }
+
+ private static Map<String, String> fields(final String... nameTypePairs) {
+ final Map<String, String> fields = new HashMap<>();
+ for (int i = 0; i < nameTypePairs.length; i += 2) {
+ fields.put(nameTypePairs[i], nameTypePairs[i + 1]);
+ }
+ return fields;
+ }
+
+ /** The column addition runs on the DataNode and the procedure may finish a bit later. */
+ private static void awaitFields(
+ final Connection table,
+ final String viewDb,
+ final String view,
+ final Map<String, String> expected)
+ throws Exception {
+ final long deadline = System.currentTimeMillis() + 10_000L;
+ Map<String, String> actual = fieldColumns(table, viewDb, view);
+ while (!expected.equals(actual) && System.currentTimeMillis() < deadline) {
+ Thread.sleep(100L);
+ actual = fieldColumns(table, viewDb, view);
+ }
+ Assert.assertEquals(expected, actual);
+ }
+
+ /** A column which must not be added; waits a little so that a late addition is caught too. */
+ private static void awaitAbsent(
+ final Connection table, final String viewDb, final String view, final String columnName)
+ throws Exception {
+ Thread.sleep(500L);
+ Assert.assertFalse(
+ "Column " + columnName + " should not be added to " + view,
+ fieldColumns(table, viewDb, view).containsKey(columnName));
+ }
+}
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 6df31101e47..36d5d2e8aff 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -52,6 +52,7 @@ public enum TSStatusCode {
DISPATCH_ERROR(306),
LICENSE_ERROR(307),
THRIFT_FRAME_OVERSIZE(308),
+ REPEATED_WITH_EXISTING_TASK(309),
// Client,
REDIRECTION_RECOMMEND(400),
diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
index 183ab36c478..482da8daad1 100644
--- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
@@ -614,6 +614,9 @@ public final class ManagerMessages {
public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "Got IOException when construct database Tree";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "Got IOException when deserialize template info";
public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "Measurements not found for %s, cannot auto detect";
+ public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "The duplicated source measurement %s is unsupported yet.";
+ public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "Multiple types encountered when auto detecting type of measurement '%s', please check";
+ public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "The measurements %s and %s share the same lower case when auto detecting type, please check";
public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "Failed to take snapshot, because snapshot file [{}] is already exist.";
public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "Failed to load snapshot,snapshot file [{}] is not exist.";
public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "you maybe need to rename the simple dir to 0_0 manually.";
diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
index 7154b0b4c82..488e7cc7598 100644
--- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
@@ -490,7 +490,8 @@ public final class ProcedureMessages {
"Failed to rollback pipe plugin [%s] on data nodes";
public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_FOR_TABLE_INFO_TO_DATANODE =
"Failed to rollback pre-release {} for table {}.{} info to DataNode, failure results: {}";
- public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED = "Failed to prove an unreachable DataNode is fenced";
+ public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED =
+ "Failed to prove an unreachable DataNode is fenced";
public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_TEMPLATE_INFO_OF_TEMPLATE_SET =
"Failed to rollback pre release template info of template {} set on path {} on DataNode {}";
public static final String FAILED_TO_ROLLBACK_PRE_SET_TEMPLATE_ON_PATH_DUE_TO =
@@ -1435,10 +1436,16 @@ public final class ProcedureMessages {
"rolling back after failure: %s.";
public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 =
"rolling back after an earlier failure.";
+ public static final String
+ MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F =
+ "Some other task is setting or unsetting a template on target path %s.";
public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F =
"DataNode setSystemStatus RPC failed: %s: %s: %s; ";
public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 =
"ConfigNode trySetNodeStatus failed: %s: %s: %s; ";
+ public static final String
+ LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 =
+ "Failed to detect the tree series under the new view {}.{}: {}";
public static final String
LOG_PROCEDURE_ARG_IS_PAST_ITS_COMMIT_POINT_KEEPING_EVERY_STATE_IT_EXECUTED_CA21D7E9 =
"Procedure {} is past its commit point; keeping every state it executed.";
diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
index fa3d63dddb3..739ee0029c8 100644
--- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
@@ -606,6 +606,9 @@ public final class ManagerMessages {
public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "构建数据库树时发生 IOException";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "反序列化模板信息时发生 IOException";
public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "未找到 %s 的测点,无法自动检测";
+ public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "暂不支持重复的源测点 %s。";
+ public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "自动检测测点 %s 的类型时遇到多种类型,请检查";
+ public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "自动检测类型时,测点 %s 和 %s 的小写形式相同,请检查";
public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "获取快照失败,原因:快照文件 [{}] 已存在。";
public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "无法加载快照,快照文件 [{}] 不存在。";
public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "可能需要手动将 simple 目录重命名为 0_0。";
diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
index 973241cd431..d65ee58ec53 100644
--- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
@@ -1520,10 +1520,16 @@ public final class ProcedureMessages {
public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚,失败原因:%s。";
public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 =
"正在回滚此前发生的失败。";
+ public static final String
+ MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F =
+ "有其他任务正在目标路径 %s 上设置或卸载模板。";
public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F =
"DataNode setSystemStatus RPC 失败:%s:%s:%s; ";
public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 =
"ConfigNode trySetNodeStatus 失败:%s:%s:%s; ";
+ public static final String
+ LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 =
+ "扫描新建视图 {}.{} 下的树序列失败:{}";
public static final String
LOG_PROCEDURE_ARG_IS_PAST_ITS_COMMIT_POINT_KEEPING_EVERY_STATE_IT_EXECUTED_CA21D7E9 =
"流程 {} 已越过提交点,保留它执行过的所有状态。";
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index f82d8187fab..a08b4e87ba1 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -158,6 +158,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
@@ -2286,7 +2288,7 @@ public class ConfigManager implements IManager {
}
@Override
- public synchronized TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) {
+ public TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) {
TSStatus status = confirmLeader();
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return procedureManager.setSchemaTemplate(
@@ -2368,7 +2370,7 @@ public class ConfigManager implements IManager {
}
@Override
- public synchronized TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) {
+ public TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) {
TSStatus status = confirmLeader();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return status;
@@ -3378,6 +3380,14 @@ public class ConfigManager implements IManager {
}
}
+ @Override
+ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) {
+ final TSStatus status = confirmLeader();
+ return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ ? procedureManager.autoSyncViewColumns(req)
+ : new TAutoSyncViewColumnsResp(status);
+ }
+
@Override
public TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req) {
final TSStatus status = confirmLeader();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index 37cc7c72247..9d2ea4c725d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -72,6 +72,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
@@ -919,6 +921,8 @@ public interface IManager {
TSStatus alterOrDropTable(final TAlterOrDropTableReq req);
+ TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req);
+
TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req);
TShowTableResp showTables(final String database, final boolean isDetails);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 18f65a98481..62b90a9845f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
+import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType;
import org.apache.iotdb.commons.schema.table.TableNodeStatus;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
@@ -151,6 +152,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
@@ -192,6 +196,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -201,6 +206,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -216,6 +222,15 @@ public class ProcedureManager {
private static final String PROCEDURE_TIMEOUT_MESSAGE =
"Timed out to wait for procedure return. The procedure is still running.";
+ /** One addColumnForTree2TableViews call waits at most this long for its procedures to finish. */
+ private static final long ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS = 30_000L;
+
+ /** How many times a request which overlapped with another task is submitted again. */
+ private static final int OVERLAP_RETRY_ROUNDS = 4;
+
+ /** How long a retry round waits for the overlapping task to finish before submitting again. */
+ private static final long OVERLAP_RETRY_WAIT_MS = 2_000L;
+
private final ConfigManager configManager;
private ProcedureExecutor<ConfigNodeProcedureEnv> executor;
private ProcedureScheduler scheduler;
@@ -542,17 +557,27 @@ public class ProcedureManager {
SetTemplateProcedure setTemplateProcedure;
for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
+ if (Objects.isNull(type)) {
continue;
}
- setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
- if (queryId.equals(setTemplateProcedure.getQueryId())) {
- procedure = setTemplateProcedure;
- break;
- }
- if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
- hasOverlappedTask = true;
- break;
+ if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
+ setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
+ if (queryId.equals(setTemplateProcedure.getQueryId())) {
+ procedure = setTemplateProcedure;
+ break;
+ }
+ if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ } else if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
+ // A path is set with one template at a time, so setting and unsetting the same path
+ // overlap as well.
+ if (templateSetPath.equals(
+ ((UnsetTemplateProcedure) runningProcedure).getPath().getFullPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
}
}
@@ -560,7 +585,10 @@ public class ProcedureManager {
if (hasOverlappedTask) {
return RpcUtils.getStatus(
TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is setting template on target path.");
+ String.format(
+ ProcedureMessages
+ .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F,
+ templateSetPath));
}
procedure =
new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe);
@@ -626,18 +654,28 @@ public class ProcedureManager {
UnsetTemplateProcedure unsetTemplateProcedure;
for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
+ if (Objects.isNull(type)) {
continue;
}
- unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
- if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
- procedure = unsetTemplateProcedure;
- break;
- }
- if (template.getId() == unsetTemplateProcedure.getTemplateId()
- && path.equals(unsetTemplateProcedure.getPath())) {
- hasOverlappedTask = true;
- break;
+ if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
+ unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
+ if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
+ procedure = unsetTemplateProcedure;
+ break;
+ }
+ if (template.getId() == unsetTemplateProcedure.getTemplateId()
+ && path.equals(unsetTemplateProcedure.getPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ } else if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
+ // A path is set with one template at a time, so setting and unsetting the same path
+ // overlap as well.
+ if (path.getFullPath()
+ .equals(((SetTemplateProcedure) runningProcedure).getTemplateSetPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
}
}
@@ -645,8 +683,10 @@ public class ProcedureManager {
if (hasOverlappedTask) {
return RpcUtils.getStatus(
TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is unsetting target template from target path "
- + path.getFullPath());
+ String.format(
+ ProcedureMessages
+ .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F,
+ path.getFullPath()));
}
procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe);
this.executor.submitProcedure(procedure);
@@ -2269,6 +2309,167 @@ public class ProcedureManager {
false));
}
+ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) {
+ if (req == null || !req.isSetViewInfos()) {
+ return new TAutoSyncViewColumnsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER));
+ }
+
+ final List<TAutoSyncViewColumnsFailure> failedViews = new ArrayList<>();
+ final List<Pair<TAlterOrDropTableReq, Procedure<ConfigNodeProcedureEnv>>> procedures =
+ new ArrayList<>();
+ // The requests which overlapped with another task on the same table, together with the status
+ // of the last submission attempt, see processOverlappingTask.
+ final List<Pair<TAlterOrDropTableReq, TSStatus>> overlappingReqs = new ArrayList<>();
+
+ // Submit every view before waiting, so procedures for different views can run concurrently.
+ for (final TAlterOrDropTableReq addColumnReq : req.getViewInfos()) {
+ if (!isAddViewColumnReq(addColumnReq)) {
+ failedViews.add(illegalAddViewColumnFailure(addColumnReq));
+ continue;
+ }
+
+ final TSStatus submissionStatus = submitAddViewColumn(addColumnReq, procedures, failedViews);
+ if (submissionStatus.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) {
+ overlappingReqs.add(new Pair<>(addColumnReq, submissionStatus));
+ }
+ }
+
+ processOverlappingTask(overlappingReqs, procedures, failedViews);
+
+ final long waitDeadline = System.currentTimeMillis() + ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS;
+ for (final Pair<TAlterOrDropTableReq, Procedure<ConfigNodeProcedureEnv>> procedure :
+ procedures) {
+ final long remainingWaitTime = Math.max(0, waitDeadline - System.currentTimeMillis());
+ final TSStatus status = waitingProcedureFinished(procedure.getRight(), remainingWaitTime);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ failedViews.add(
+ new TAutoSyncViewColumnsFailure(
+ procedure.getLeft().database, procedure.getLeft().tableName, status));
+ }
+ }
+
+ final TSStatus status = failedViews.isEmpty() ? StatusUtils.OK : StatusUtils.INTERNAL_ERROR;
+ final TAutoSyncViewColumnsResp response = new TAutoSyncViewColumnsResp(status);
+ if (!failedViews.isEmpty()) {
+ response.setFailedViews(failedViews);
+ }
+ return response;
+ }
+
+ /** Whether the request carries everything an add-column operation on a view needs. */
+ private static boolean isAddViewColumnReq(final @Nullable TAlterOrDropTableReq addColumnReq) {
+ return Objects.nonNull(addColumnReq)
+ && addColumnReq.isSetDatabase()
+ && addColumnReq.isSetTableName()
+ && addColumnReq.isSetQueryId()
+ && addColumnReq.isSetOperationType()
+ && addColumnReq.getOperationType()
+ == AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue()
+ && addColumnReq.isSetUpdateInfo()
+ && addColumnReq.isSetIsView()
+ && addColumnReq.isIsView();
+ }
+
+ private static TAutoSyncViewColumnsFailure illegalAddViewColumnFailure(
+ final TAlterOrDropTableReq addColumnReq) {
+ return new TAutoSyncViewColumnsFailure(
+ Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getDatabase(), ""),
+ Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getTableName(), ""),
+ RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER));
+ }
+
+ /**
+ * Submits the add-column procedure of one view and settles everything but an overlap with another
+ * task on the same table.
+ */
+ private TSStatus submitAddViewColumn(
+ final TAlterOrDropTableReq addColumnReq,
+ final List<Pair<TAlterOrDropTableReq, Procedure<ConfigNodeProcedureEnv>>> procedures,
+ final List<TAutoSyncViewColumnsFailure> failedViews) {
+ final AddViewColumnProcedure procedure;
+ procedure =
+ new AddViewColumnProcedure(
+ addColumnReq.database,
+ addColumnReq.tableName,
+ addColumnReq.queryId,
+ Collections.emptyList(),
+ false);
+
+ final Pair<TSStatus, Long> submissionResult =
+ executeWithoutDuplicateWithoutWaiting(
+ addColumnReq.database,
+ null,
+ addColumnReq.tableName,
+ null,
+ addColumnReq.queryId,
+ ProcedureType.ADD_VIEW_COLUMN_PROCEDURE,
+ procedure,
+ proc -> {
+ if (proc instanceof AddViewColumnProcedure) {
+ ((AddViewColumnProcedure) proc)
+ .setAddedColumnList(
+ TsTableColumnSchemaUtil.deserializeColumnSchemaList(
+ addColumnReq.updateInfo));
+ }
+ });
+ final TSStatus status = submissionResult.getLeft();
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ procedures.add(new Pair<>(addColumnReq, procedure));
+ } else if (status.getCode() == TSStatusCode.REPEATED_WITH_EXISTING_TASK.getStatusCode()) {
+ final Procedure<ConfigNodeProcedureEnv> repeatedProcedure =
+ executor.getProcedures().get(submissionResult.getRight());
+ // null means the procedure could have finished at this moment
+ if (Objects.isNull(repeatedProcedure)) {
+ failedViews.add(
+ new TAutoSyncViewColumnsFailure(
+ addColumnReq.database,
+ addColumnReq.tableName,
+ RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)));
+ } else {
+ procedures.add(new Pair<>(addColumnReq, repeatedProcedure));
+ }
+ } else if (status.getCode() != TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) {
+ failedViews.add(
+ new TAutoSyncViewColumnsFailure(addColumnReq.database, addColumnReq.tableName, status));
+ }
+ return status;
+ }
+
+ /**
+ * Retries the requests which overlapped with another task on the same table, A request which
+ * still overlaps after the last round is reported as a failure, with the status of its last
+ * attempt.
+ */
+ private void processOverlappingTask(
+ final List<Pair<TAlterOrDropTableReq, TSStatus>> overlappingReqs,
+ final List<Pair<TAlterOrDropTableReq, Procedure<ConfigNodeProcedureEnv>>> procedures,
+ final List<TAutoSyncViewColumnsFailure> failedViews) {
+ List<Pair<TAlterOrDropTableReq, TSStatus>> pending = overlappingReqs;
+ for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) {
+ try {
+ Thread.sleep(OVERLAP_RETRY_WAIT_MS);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ final List<Pair<TAlterOrDropTableReq, TSStatus>> stillOverlapping =
+ new ArrayList<>(pending.size());
+ for (final Pair<TAlterOrDropTableReq, TSStatus> request : pending) {
+ final TSStatus status = submitAddViewColumn(request.getLeft(), procedures, failedViews);
+ if (status.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) {
+ stillOverlapping.add(new Pair<>(request.getLeft(), status));
+ }
+ }
+ pending = stillOverlapping;
+ }
+
+ for (final Pair<TAlterOrDropTableReq, TSStatus> request : pending) {
+ failedViews.add(
+ new TAutoSyncViewColumnsFailure(
+ request.getLeft().database, request.getLeft().tableName, request.getRight()));
+ }
+ }
+
public TSStatus alterTableSetProperties(final TAlterOrDropTableReq req) {
final boolean isView = req.isSetIsView() && req.isIsView();
return executeWithoutDuplicate(
@@ -2617,6 +2818,38 @@ public class ProcedureManager {
return waitingProcedureFinished(procedureToWait);
}
+ public Pair<TSStatus, Long> executeWithoutDuplicateWithoutWaiting(
+ final String database,
+ final TsTable table,
+ final String tableName,
+ final @Nullable String newName,
+ final String queryId,
+ final ProcedureType thisType,
+ final Procedure<ConfigNodeProcedureEnv> procedure,
+ final Consumer<Procedure<ConfigNodeProcedureEnv>> consumer) {
+ final long procedureId;
+ synchronized (this) {
+ final Pair<Long, Boolean> procedureIdDuplicatePair =
+ checkDuplicateTableTask(database, table, tableName, newName, queryId, thisType);
+ procedureId = procedureIdDuplicatePair.getLeft();
+ if (procedureId == -1) {
+ if (Boolean.TRUE.equals(procedureIdDuplicatePair.getRight())) {
+ TSStatus status =
+ RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is operating table with same name.");
+ return new Pair<>(status, -1L);
+ }
+ consumer.accept(procedure);
+ this.executor.submitProcedure(procedure);
+ return new Pair<>(StatusUtils.OK, procedure.getProcId());
+ } else {
+ TSStatus status = RpcUtils.getStatus(TSStatusCode.REPEATED_WITH_EXISTING_TASK, "");
+ return new Pair<>(status, procedureId);
+ }
+ }
+ }
+
public Pair<Long, Boolean> checkDuplicateTableTask(
final @Nonnull String database,
final TsTable table,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
index d85e2e81013..88121ed6ae6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
@@ -1653,7 +1653,7 @@ public class ClusterSchemaManager {
}
}
- public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnExtension(
+ public Pair<TSStatus, TsTable> tableColumnCheckForColumnExtension(
final String database,
final String tableName,
final List<TsTableColumnSchema> columnSchemaList,
@@ -1676,8 +1676,7 @@ public class ClusterSchemaManager {
return result.get();
}
- final TsTable expandedTable = new TsTable(originalTable);
-
+ // the table read above is a copy, which this check expands and returns
final String errorMsg =
String.format(
"Column '%s' already exist",
@@ -1686,29 +1685,74 @@ public class ClusterSchemaManager {
.collect(Collectors.joining(", ")));
final AtomicBoolean hasObject = new AtomicBoolean(false);
+ if (originalTable.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).isEmpty()) {
+ checkColumnSchema(originalTable, columnSchemaList, hasObject);
+ } else {
+ // only for tree_to_table view
+ checkColumnSchemaForTree2TableView(originalTable, columnSchemaList, hasObject);
+ }
+
+ if (columnSchemaList.isEmpty()) {
+ return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null);
+ }
+
+ if (hasObject.get()) {
+ originalTable.checkTableNameAndObjectNames4Object();
+ }
+ return new Pair<>(StatusUtils.OK, originalTable);
+ }
+
+ private static void checkColumnSchema(
+ TsTable table, List<TsTableColumnSchema> columnSchemaList, AtomicBoolean hasObject) {
columnSchemaList.removeIf(
columnSchema -> {
- if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) {
+ if (Objects.isNull(table.getColumnSchema(columnSchema.getColumnName()))) {
if (columnSchema.getDataType().equals(TSDataType.OBJECT)) {
hasObject.set(true);
}
- expandedTable.addColumnSchema(columnSchema);
+ table.addColumnSchema(columnSchema);
return false;
}
return true;
});
+ }
- if (columnSchemaList.isEmpty()) {
- return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null);
- }
+ private static void checkColumnSchemaForTree2TableView(
+ final TsTable view,
+ final List<TsTableColumnSchema> columnSchemaList,
+ final AtomicBoolean hasObject) {
+ Set<String> existingColumnNames =
+ view.getColumnList().stream()
+ .map(TsTableColumnSchema::getColumnName)
+ .collect(Collectors.toSet());
+ final Set<String> originalNames = TreeViewSchema.getSourceNames(view);
- if (hasObject.get()) {
- expandedTable.checkTableNameAndObjectNames4Object();
- }
- return new Pair<>(StatusUtils.OK, expandedTable);
+ columnSchemaList.removeIf(
+ columnSchema -> {
+ if (existingColumnNames.contains(columnSchema.getColumnName())) {
+ // todo log
+ return true;
+ }
+
+ // The original time series can only be referenced once.
+ final String originalName = TreeViewSchema.getSourceName(columnSchema);
+ if (originalNames.contains(originalName)) {
+ // todo log
+ return true;
+ }
+
+ if (columnSchema.getDataType().equals(TSDataType.OBJECT)) {
+ hasObject.set(true);
+ }
+
+ existingColumnNames.add(columnSchema.getColumnName());
+ originalNames.add(originalName);
+ view.addColumnSchema(columnSchema);
+ return false;
+ });
}
- public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnAltering(
+ public Pair<TSStatus, TsTable> tableColumnCheckForColumnAltering(
final String database,
final String tableName,
final String columnName,
@@ -1734,8 +1778,8 @@ public class ClusterSchemaManager {
return new Pair<>(tsStatus, null);
}
- final TsTable alteredTable = new TsTable(originalTable);
- TsTableColumnSchema tsTableColumnSchema = alteredTable.getColumnSchema(columnName);
+ // the table read above is a copy, which this check alters and returns
+ TsTableColumnSchema tsTableColumnSchema = originalTable.getColumnSchema(columnName);
tsTableColumnSchema.setDataType(dataType);
if (tsTableColumnSchema instanceof FieldColumnSchema) {
FieldColumnSchema fieldColumnSchema = ((FieldColumnSchema) tsTableColumnSchema);
@@ -1743,10 +1787,10 @@ public class ClusterSchemaManager {
SchemaUtils.getDataTypeCompatibleEncoding(dataType, fieldColumnSchema.getEncoding()));
}
- return new Pair<>(RpcUtils.SUCCESS_STATUS, alteredTable);
+ return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable);
}
- public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnRenaming(
+ public Pair<TSStatus, TsTable> tableColumnCheckForColumnRenaming(
final String database,
final String tableName,
final String oldName,
@@ -1794,14 +1838,13 @@ public class ClusterSchemaManager {
null);
}
- final TsTable expandedTable = new TsTable(originalTable);
-
- expandedTable.renameColumnSchema(oldName, newName);
+ // the table read above is a copy, which this check renames and returns
+ originalTable.renameColumnSchema(oldName, newName);
- return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable);
+ return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable);
}
- public synchronized Pair<TSStatus, TsTable> tableCheckForRenaming(
+ public Pair<TSStatus, TsTable> tableCheckForRenaming(
final String database,
final String tableName,
final String newName,
@@ -1831,9 +1874,9 @@ public class ClusterSchemaManager {
null);
}
- final TsTable expandedTable = new TsTable(originalTable);
- expandedTable.renameTable(newName);
- return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable);
+ // the table read above is a copy, which this check renames and returns
+ originalTable.renameTable(newName);
+ return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable);
}
public TSStatus executePlan(final ConfigPhysicalPlan plan, final boolean isGeneratedByPipe) {
@@ -1845,7 +1888,7 @@ public class ClusterSchemaManager {
}
}
- public synchronized TSStatus setTableComment(
+ public TSStatus setTableComment(
final String database,
final String tableName,
final String comment,
@@ -1865,7 +1908,7 @@ public class ClusterSchemaManager {
}
}
- public synchronized TSStatus setTableColumnComment(
+ public TSStatus setTableColumnComment(
final String database,
final String tableName,
final String columnName,
@@ -1885,7 +1928,7 @@ public class ClusterSchemaManager {
}
}
- public synchronized Pair<TSStatus, TsTable> updateTableProperties(
+ public Pair<TSStatus, TsTable> updateTableProperties(
final String database,
final String tableName,
final Map<String, String> originalProperties,
@@ -1936,24 +1979,24 @@ public class ClusterSchemaManager {
throw new MetadataException(e);
}
- final TsTable updatedTable = new TsTable(originalTable);
+ // the table read above is a copy, which this check updates and returns
updatedProperties.forEach(
(k, v) -> {
originalProperties.put(k, originalTable.getPropValue(k).orElse(null));
if (Objects.nonNull(v)) {
- updatedTable.addProp(k, v);
+ originalTable.addProp(k, v);
} else if (TsTable.NEED_LAST_CACHE_PROPERTY.equals(k)
&& Objects.nonNull(databaseSchema)
&& databaseSchema.isSetNeedLastCache()) {
- updatedTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache()));
+ originalTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache()));
} else {
- updatedTable.removeProp(k);
+ originalTable.removeProp(k);
}
});
- return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable);
+ return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable);
}
- public synchronized Pair<TSStatus, TsTable> updateTableColumnProperties(
+ public Pair<TSStatus, TsTable> updateTableColumnProperties(
final String database,
final String tableName,
final String columnName,
@@ -1984,8 +2027,8 @@ public class ClusterSchemaManager {
null);
}
- final TsTable updatedTable = new TsTable(originalTable);
- final TsTableColumnSchema updatedColumn = updatedTable.getColumnSchema(columnName);
+ // the table read above is a copy, which this check updates and returns
+ final TsTableColumnSchema updatedColumn = originalTable.getColumnSchema(columnName);
updatedProperties
.keySet()
.removeIf(
@@ -2005,7 +2048,7 @@ public class ClusterSchemaManager {
}
}
- return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable);
+ return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable);
}
private void invalidateLastCache(final String database) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
index 94fc65e8ba2..c5736dfa4cf 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java
@@ -1550,12 +1550,23 @@ public class ClusterSchemaInfo implements SnapshotProcessor {
}
}
+ /**
+ * The table and its status, or empty when the table does not exist. The table is a copy, which
+ * the caller owns: the stored table is only written by the plans, under the write lock, and never
+ * through a reference handed out here.
+ */
public Optional<Pair<TsTable, TableNodeStatus>> getTsTableIfExists(
final String database, final String tableName) throws MetadataException {
databaseReadWriteLock.readLock().lock();
try {
- return tableModelMTree.getTableAndStatusIfExists(
- getQualifiedDatabasePartialPath(database), tableName);
+ final Optional<Pair<TsTable, TableNodeStatus>> storedTable =
+ tableModelMTree.getTableAndStatusIfExists(
+ getQualifiedDatabasePartialPath(database), tableName);
+ if (storedTable.isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.of(
+ new Pair<>(new TsTable(storedTable.get().getLeft()), storedTable.get().getRight()));
} finally {
databaseReadWriteLock.readLock().unlock();
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java
index f407d4d8ecd..2beca63c2e6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
@@ -32,6 +33,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
+import org.apache.iotdb.confignode.i18n.ProcedureMessages;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor;
import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewReq;
@@ -40,17 +42,16 @@ import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -64,28 +65,32 @@ public class TreeDeviceViewFieldDetector {
private final ConfigManager configManager;
private final PartialPath path;
private final TsTable table;
- private final Map<String, Set<FieldColumnSchema>> fields;
private TDeviceViewResp result = new TDeviceViewResp(StatusUtils.OK, new ConcurrentHashMap<>());
private final Map<String, String> lowerCase2OriginalMap = new HashMap<>();
- public TreeDeviceViewFieldDetector(
- final ConfigManager configManager,
- final TsTable table,
- final Map<String, Set<FieldColumnSchema>> fields) {
+ public TreeDeviceViewFieldDetector(final ConfigManager configManager, final TsTable table) {
this.configManager = configManager;
this.path = TreeViewSchema.getPrefixPattern(table);
this.table = table;
- this.fields = fields;
}
- public TSStatus detectMissingFieldTypes() {
- if (table.getFieldNum() == 0 && Objects.isNull(fields)) {
+ /**
+ * Fills the fields of the view which is being created: a view may declare no field at all, in
+ * which case every field under its tree path pattern becomes a column, or declare fields whose
+ * type is left to the detection.
+ */
+ public TSStatus detectFieldsForCreation() {
+ if (table.getFieldNum() == 0) {
+ // for the first time creation of the table, user do not assign any field;
+ // just detect all the fields of prefix path
new TreeDeviceViewFieldDetectionTaskExecutor(
configManager,
getLatestSchemaRegionMap(),
table.getTagNum(),
- TreeViewSchema.isRestrict(table))
+ TreeViewSchema.isRestrict(table),
+ null,
+ false)
.execute();
if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return result.getStatus();
@@ -93,69 +98,158 @@ public class TreeDeviceViewFieldDetector {
result
.getDeviewViewFieldTypeMap()
.forEach(
- (field, type) -> {
- final FieldColumnSchema columnSchema =
- new FieldColumnSchema(field, TSDataType.getTsDataType(type));
- if (!field.equals(lowerCase2OriginalMap.get(field))) {
- TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field));
- }
- table.addColumnSchema(columnSchema);
- });
- } else {
- final Map<String, Set<FieldColumnSchema>> unknownFields;
- if (Objects.isNull(fields)) {
- unknownFields = new HashMap<>();
- for (final TsTableColumnSchema schema : table.getColumnList()) {
- if (!(schema instanceof FieldColumnSchema)
- || schema.getDataType() != TSDataType.UNKNOWN) {
- continue;
- }
- final String key = TreeViewSchema.getSourceName(schema);
- if (!unknownFields.containsKey(key)) {
- unknownFields.put(key, new HashSet<>());
- }
- unknownFields.get(key).add((FieldColumnSchema) schema);
- }
- } else {
- unknownFields = fields;
- }
+ (field, type) ->
+ table.addColumnSchema(toFieldColumnSchemaWithOriginName(field, type)));
+ return StatusUtils.OK;
+ }
+
+ return detectFieldTypes(collectUnknownFieldColumns());
+ }
+
+ /**
+ * Resolves the type of the field columns which are being added to an existing view. The columns
+ * to resolve are given by the caller, as only it knows which ones it is adding.
+ */
+ public TSStatus detectFieldTypesForColumnAddition(final Map<String, FieldColumnSchema> fields) {
+ return detectFieldTypes(fields);
+ }
+
+ /**
+ * The fields which exist under the view's tree path pattern but are not columns of the view yet,
+ * as the columns to add. Used after a view is created, to pick up the series which were created
+ * before the view was mounted on the DataNodes, and which the DataNodes therefore never reported.
+ */
+ public Pair<TSStatus, List<TsTableColumnSchema>> detectNewFieldColumnsAfterCreation() {
+ new TreeDeviceViewFieldDetectionTaskExecutor(
+ configManager,
+ getLatestSchemaRegionMap(),
+ table.getTagNum(),
+ TreeViewSchema.isRestrict(table),
+ null,
+ true)
+ .execute();
+ if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return new Pair<>(result.getStatus(), Collections.emptyList());
+ }
+ // A source measurement can only be referenced once, so a field whose source the view already
+ // references under another column name gets no column either.
+ final Set<String> sourceNames = TreeViewSchema.getSourceNames(table);
+ final List<TsTableColumnSchema> columns = new ArrayList<>();
+ result
+ .getDeviewViewFieldTypeMap()
+ .forEach(
+ (field, type) -> {
+ if (Objects.isNull(table.getColumnSchema(field))
+ && !sourceNames.contains(lowerCase2OriginalMap.get(field))) {
+ columns.add(toFieldColumnSchemaWithOriginName(field, type));
+ }
+ });
+ return new Pair<>(StatusUtils.OK, columns);
+ }
+
+ /**
+ * The columns which are missing from a view which auto-syncs its schema: the series which already
+ * existed under its tree path pattern were created before the DataNodes mounted the view on their
+ * tree-to-table view index, so they never triggered the addition of their column. Used right
+ * after the view is mounted, whether it has just been created or its auto detection has just been
+ * turned on.
+ *
+ * @return the columns to add; empty when the view does not auto-sync its schema, or when the
+ * detection itself failed
+ */
+ public static List<TsTableColumnSchema> detectMissingColumnsAfterCreation(
+ final ConfigManager configManager, final String database, final TsTable table) {
+ final boolean autoSync =
+ table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false);
+ if (!autoSync) {
+ return Collections.emptyList();
+ }
- if (unknownFields.isEmpty()) {
- return StatusUtils.OK;
+ final Pair<TSStatus, List<TsTableColumnSchema>> detected =
+ new TreeDeviceViewFieldDetector(configManager, table).detectNewFieldColumnsAfterCreation();
+ if (detected.getLeft().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ LOGGER.warn(
+ ProcedureMessages
+ .LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7,
+ database,
+ table.getTableName(),
+ detected.getLeft());
+ return Collections.emptyList();
+ }
+ return detected.getRight();
+ }
+
+ /** The field columns of the view whose type is not known yet, keyed by their source name. */
+ private Map<String, FieldColumnSchema> collectUnknownFieldColumns() {
+ final Map<String, FieldColumnSchema> unknownFields = new HashMap<>();
+ for (final TsTableColumnSchema schema : table.getColumnList()) {
+ if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) {
+ continue;
}
- new TreeDeviceViewFieldDetectionTaskExecutor(
- configManager,
- getLatestSchemaRegionMap(),
- table.getTagNum(),
- TreeViewSchema.isRestrict(table),
- unknownFields.keySet())
- .execute();
- if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return result.getStatus();
+ final String key = TreeViewSchema.getSourceName(schema);
+ if (unknownFields.containsKey(key)) {
+ // A table which references the same source measurement twice is rejected when it is
+ // parsed, so a table in the schema must not reach this point.
+ throw new SemanticException(
+ String.format(
+ ManagerMessages
+ .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566,
+ key));
}
- for (final Map.Entry<String, Set<FieldColumnSchema>> unknownField :
- unknownFields.entrySet()) {
- if (result.getDeviewViewFieldTypeMap().containsKey(unknownField.getKey())) {
- unknownField
- .getValue()
- .forEach(
- field ->
- field.setDataType(
- TSDataType.getTsDataType(
- result.getDeviewViewFieldTypeMap().get(unknownField.getKey()))));
- } else {
- return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode())
- .setMessage(
- String.format(
- ManagerMessages
- .MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44,
- unknownField.getKey()));
- }
+ unknownFields.put(key, (FieldColumnSchema) schema);
+ }
+ return unknownFields;
+ }
+
+ /**
+ * Resolves the type of every field column in {@code unknownFields} from the schema of the regions
+ * the view covers. A measurement which the detection cannot find fails the whole detection.
+ */
+ private TSStatus detectFieldTypes(final Map<String, FieldColumnSchema> unknownFields) {
+ if (unknownFields.isEmpty()) {
+ return StatusUtils.OK;
+ }
+ new TreeDeviceViewFieldDetectionTaskExecutor(
+ configManager,
+ getLatestSchemaRegionMap(),
+ table.getTagNum(),
+ TreeViewSchema.isRestrict(table),
+ unknownFields.keySet(),
+ false)
+ .execute();
+ if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return result.getStatus();
+ }
+ for (final Map.Entry<String, FieldColumnSchema> unknownField : unknownFields.entrySet()) {
+ // The detection keys the field types by the lower case of the measurement name, while the
+ // column keeps the source name as the statement spelled it.
+ final String sourceName = unknownField.getKey().toLowerCase(Locale.ENGLISH);
+ if (result.getDeviewViewFieldTypeMap().containsKey(sourceName)) {
+ unknownField
+ .getValue()
+ .setDataType(
+ TSDataType.getTsDataType(result.getDeviewViewFieldTypeMap().get(sourceName)));
+ } else {
+ return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode())
+ .setMessage(
+ String.format(
+ ManagerMessages.MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44,
+ unknownField.getKey()));
}
}
return StatusUtils.OK;
}
+ private TsTableColumnSchema toFieldColumnSchemaWithOriginName(
+ final String field, final byte type) {
+ final FieldColumnSchema columnSchema =
+ new FieldColumnSchema(field, TSDataType.getTsDataType(type));
+ if (!field.equals(lowerCase2OriginalMap.get(field))) {
+ TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field));
+ }
+ return columnSchema;
+ }
+
private Map<TConsensusGroupId, TRegionReplicaSet> getLatestSchemaRegionMap() {
final PathPatternTree tree = new PathPatternTree();
tree.appendPathPattern(path);
@@ -166,59 +260,37 @@ public class TreeDeviceViewFieldDetector {
private class TreeDeviceViewFieldDetectionTaskExecutor
extends DataNodeRegionTaskExecutor<TDeviceViewReq, TDeviceViewResp> {
- protected TreeDeviceViewFieldDetectionTaskExecutor(
- final ConfigManager configManager,
- final Map<TConsensusGroupId, TRegionReplicaSet> targetRegionGroup,
- final int tagNumber,
- final boolean restrict) {
- super(
- configManager,
- targetRegionGroup,
- false,
- CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE,
- ((dataNodeLocation, consensusGroupIdList) ->
- new TDeviceViewReq(
- consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict)));
- configManager
- .getClusterSchemaManager()
- .getTemplateSetInfo(Collections.singletonList(path))
- .getPatternTemplateMap()
- .values()
- .stream()
- .flatMap(Collection::stream)
- .flatMap(template -> template.getSchemaMap().values().stream())
- .forEach(
- schema ->
- mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte()));
- }
+ /**
+ * Whether several types of the same measurement are tolerated. When they are, the type the
+ * detection meets first is kept instead of failing the whole detection.
+ */
+ private final boolean tolerateTypeMismatch;
protected TreeDeviceViewFieldDetectionTaskExecutor(
final ConfigManager configManager,
final Map<TConsensusGroupId, TRegionReplicaSet> targetRegionGroup,
final int tagNumber,
final boolean restrict,
- final @Nonnull Set<String> measurements) {
+ final @Nullable Set<String> measurements,
+ final boolean tolerateTypeMismatch) {
super(
configManager,
targetRegionGroup,
false,
CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE,
- ((dataNodeLocation, consensusGroupIdList) ->
- new TDeviceViewReq(
- consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict)
- .setRequiredMeasurements(measurements)));
- configManager
- .getClusterSchemaManager()
- .getTemplateSetInfo(Collections.singletonList(path))
- .getPatternTemplateMap()
- .values()
- .stream()
- .flatMap(Collection::stream)
- .flatMap(template -> template.getSchemaMap().values().stream())
- .filter(schema -> measurements.contains(schema.getMeasurementName()))
- .forEach(
- schema ->
- mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte()));
+ ((dataNodeLocation, consensusGroupIdList) -> {
+ final TDeviceViewReq req =
+ new TDeviceViewReq(
+ consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict);
+ if (Objects.nonNull(measurements)) {
+ req.setRequiredMeasurements(measurements);
+ }
+ if (tolerateTypeMismatch) {
+ req.setTolerateTypeMismatch(true);
+ }
+ return req;
+ }));
+ this.tolerateTypeMismatch = tolerateTypeMismatch;
}
@Override
@@ -273,25 +345,30 @@ public class TreeDeviceViewFieldDetector {
// Field type collection
if (!result.getDeviewViewFieldTypeMap().containsKey(fieldName)) {
result.getDeviewViewFieldTypeMap().put(fieldName, type);
- } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type)) {
+ } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type)
+ && !tolerateTypeMismatch) {
result.setStatus(
RpcUtils.getStatus(
TSStatusCode.DATA_TYPE_MISMATCH,
String.format(
- "Multiple types encountered when auto detecting type of measurement '%s', please check",
+ ManagerMessages
+ .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB,
measurement)));
}
// Field name detection
if (!lowerCase2OriginalMap.containsKey(fieldName)) {
lowerCase2OriginalMap.put(fieldName, measurement);
- } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement)) {
+ } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement)
+ && !tolerateTypeMismatch) {
result.setStatus(
RpcUtils.getStatus(
TSStatusCode.MEASUREMENT_NAME_CONFLICT,
String.format(
- "The measurements %s and %s share the same lower case when auto detecting type, please check",
- lowerCase2OriginalMap.get(fieldName), measurement)));
+ ManagerMessages
+ .MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062,
+ lowerCase2OriginalMap.get(fieldName),
+ measurement)));
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
index 241eea8a702..ad01ffc3991 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
@@ -48,6 +48,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateReq;
import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp;
import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq;
import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -272,6 +273,21 @@ public class SchemaUtils {
return clientHandler.getResponseMap();
}
+ /**
+ * Broadcasts a table change at commit-release through {@link ClusterCachePropagator}: proceed
+ * once every unreachable DataNode is provably self-fenced, instead of failing on the first one. A
+ * fenced DataNode fails closed and resyncs on lease recovery, which rebuilds the tree-to-table
+ * view index it missed; a DataNode which is not provably fenced may still serve clients, so the
+ * commit is reported as unproven instead.
+ *
+ * @return whether it is safe to proceed
+ */
+ public static boolean commitReleaseTableWithHA(
+ final TUpdateTableReq req, final ConfigManager configManager) {
+ return new ClusterCachePropagator(filterFencedDataNode(configManager))
+ .propagate(targets -> broadcastTableUpdate(req, targets));
+ }
+
private static Map<Integer, TSStatus> failedOnly(final Map<Integer, TSStatus> responses) {
return responses.entrySet().stream()
.filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode())
@@ -308,24 +324,22 @@ public class SchemaUtils {
return req;
}
- public static Map<Integer, TSStatus> commitReleaseTable(
+ /**
+ * Builds the COMMIT_UPDATE_TABLE request of a change which also affects the tree-to-table view
+ * index of the DataNodes.
+ *
+ * <p>The caller declares the view attributes the DataNode needs to apply the change, so that a
+ * DataNode never reads them from the table it caches: that cache may hold a placeholder, in which
+ * case the DataNode could not compute the entry at all.
+ */
+ public static TUpdateTableReq buildCommitUpdateTableReq(
final String database,
final String tableName,
- final ConfigManager configManager,
- final @Nullable String oldName) {
+ final @Nullable String oldName,
+ final TUpdateTree2TableView viewInfo) {
final TUpdateTableReq req = buildCommitUpdateTableReq(database, tableName, oldName);
-
- final DataNodeAsyncRequestContext<TUpdateTableReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.UPDATE_TABLE, req, filterFencedDataNode(configManager));
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequest(
- clientHandler,
- ClusterCachePropagator.BROADCAST_RPC_RETRY,
- ClusterCachePropagator.BROADCAST_RPC_TIMEOUT_MS);
- return clientHandler.getResponseMap().entrySet().stream()
- .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode())
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ req.setUpdateTree2TableViewInfo(viewInfo);
+ return req;
}
/** Build the ROLLBACK_UPDATE_TABLE request used to roll back a pre-released table change. */
@@ -345,6 +359,16 @@ public class SchemaUtils {
return req;
}
+ public static TUpdateTableReq rollbackUpdateTableReq(
+ final String database,
+ final String tableName,
+ final String oldName,
+ final TUpdateTree2TableView viewInfo) {
+ final TUpdateTableReq req = rollbackUpdateTableReq(database, tableName, oldName);
+ req.setUpdateTree2TableViewInfo(viewInfo);
+ return req;
+ }
+
/**
* Broadcast an INVALIDATE_MATCHED_SCHEMA_CACHE to all DataNodes through {@link
* ClusterCachePropagator}: proceed once every unreachable DataNode is provably self-fenced (it
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
index b4aab9949c2..510519110f9 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
@@ -118,31 +117,22 @@ public abstract class AbstractAlterOrDropTableProcedure<T>
}
}
- protected void commitRelease(final ConfigNodeProcedureEnv env) {
- commitRelease(env, null);
+ /**
+ * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table
+ * view index of the DataNodes overrides this to declare the effect.
+ */
+ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) {
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName);
}
- protected void commitRelease(final ConfigNodeProcedureEnv env, final @Nullable String oldName) {
- final Map<Integer, TSStatus> failedResults =
- SchemaUtils.commitReleaseTable(
- database, table.getTableName(), env.getConfigManager(), oldName);
- if (!failedResults.isEmpty()) {
- LOGGER.warn(
- ProcedureMessages.FAILED_TO_FOR_TABLE_TO_DATANODE_FAILURE_RESULTS,
- getActionMessage(),
- database,
- table.getTableName(),
- failedResults);
- }
+ protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) {
+ commitReleaseWithHA(env, null);
}
protected void commitReleaseWithHA(
final ConfigNodeProcedureEnv env, final @Nullable String oldName) {
- final TUpdateTableReq req =
- SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName);
- final boolean proceeded =
- new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager()))
- .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets));
+ final TUpdateTableReq req = buildCommitReleaseTableReq(oldName);
+ final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager());
if (!proceeded) {
LOGGER.warn(
@@ -165,11 +155,11 @@ public abstract class AbstractAlterOrDropTableProcedure<T>
}
protected void rollbackPreRelease(
- final ConfigNodeProcedureEnv env, final @Nullable String tableName) {
+ final ConfigNodeProcedureEnv env, final @Nullable String oldName) {
// A down DataNode must not block rollback either: proceed past provably-fenced DataNodes (which
// resync on recovery) and only fail on an unacked DataNode that is not provably fenced.
final TUpdateTableReq req =
- SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), tableName);
+ SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), oldName);
final boolean proceeded =
new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager()))
.propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets));
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java
index a354f0ff3a3..79d69a17997 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java
@@ -101,7 +101,7 @@ public abstract class AbstractSetPropertiesProcedure
LOGGER.info(ProcedureMessages.SET_PROPERTIES_TO_TABLE, database, tableName);
break;
case COMMIT_RELEASE:
- commitRelease(env);
+ commitReleaseWithHA(env);
LOGGER.info(
ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_SETTING_PROPERTIES,
database,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
index 07bdbf44c55..61d35137cf8 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java
@@ -90,7 +90,7 @@ public class AddTableColumnProcedure
ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ADDING_COLUMN,
database,
tableName);
- commitRelease(env);
+ commitReleaseWithHA(env);
return Flow.NO_MORE_STATE;
default:
setFailure(
@@ -259,6 +259,10 @@ public class AddTableColumnProcedure
&& Objects.equals(this.addedColumnList, ((AddTableColumnProcedure) o).addedColumnList);
}
+ public void setAddedColumnList(List<TsTableColumnSchema> addedColumnList) {
+ this.addedColumnList = addedColumnList;
+ }
+
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), addedColumnList);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java
index 1cfda63fcf7..3a17a7de5d7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java
@@ -103,7 +103,7 @@ public class AlterTableColumnDataTypeProcedure
ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ALTERING_COLUMN,
database,
tableName);
- commitRelease(env);
+ commitReleaseWithHA(env);
return Flow.NO_MORE_STATE;
default:
setFailure(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java
index ac573463ebb..5737b433150 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java
@@ -47,7 +47,6 @@ import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map;
import java.util.Objects;
import static org.apache.iotdb.rpc.TSStatusCode.TABLE_ALREADY_EXISTS;
@@ -98,6 +97,7 @@ public class CreateTableProcedure
case COMMIT_RELEASE:
LOGGER.info(ProcedureMessages.COMMIT_RELEASE_TABLE, database, table.getTableName());
commitReleaseTable(env);
+ reconcileTree2TableViewColumns(env);
return Flow.NO_MORE_STATE;
default:
setFailure(
@@ -156,13 +156,13 @@ public class CreateTableProcedure
}
}
- private void preReleaseTable(final ConfigNodeProcedureEnv env) {
+ protected void preReleaseTable(final ConfigNodeProcedureEnv env) {
// Broadcast the pre-update to all DataNodes. Instead of failing whenever any DataNode is
// unreachable, proceed once every unacked DataNode is provably self-fenced: such a DataNode
// fails closed on its (now-stale) table cache and resyncs on lease recovery, so it cannot serve
// dirty schema. Only fail if an unacked DataNode is not provably fenced (it may still be
// serving clients).
- final TUpdateTableReq req = SchemaUtils.buildPreUpdateTableReq(database, table, null);
+ final TUpdateTableReq req = buildPreReleaseTableReq();
final boolean proceeded =
new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager()))
.propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets));
@@ -181,6 +181,10 @@ public class CreateTableProcedure
setNextState(CreateTableState.COMMIT_CREATE);
}
+ protected TUpdateTableReq buildPreReleaseTableReq() {
+ return SchemaUtils.buildPreUpdateTableReq(database, table, null);
+ }
+
private void commitCreateTable(final ConfigNodeProcedureEnv env) {
final TSStatus status =
SchemaUtils.executeInConsensusLayer(
@@ -196,20 +200,35 @@ public class CreateTableProcedure
}
}
+ /**
+ * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table
+ * view index of the DataNodes overrides this to declare the effect.
+ */
+ protected TUpdateTableReq buildCommitReleaseTableReq() {
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null);
+ }
+
private void commitReleaseTable(final ConfigNodeProcedureEnv env) {
- final Map<Integer, TSStatus> failedResults =
- SchemaUtils.commitReleaseTable(
- database, table.getTableName(), env.getConfigManager(), null);
+ final TUpdateTableReq req = buildCommitReleaseTableReq();
+ final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager());
- if (!failedResults.isEmpty()) {
+ if (!proceeded) {
LOGGER.warn(
ProcedureMessages.FAILED_TO_SYNC_TABLE_COMMIT_CREATE_INFO_TO_DATANODE_FAILURE,
database,
table.getTableName(),
- failedResults);
+ ProcedureMessages.FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED);
}
}
+ /**
+ * Adds the columns of the tree series which already exist under a newly created tree-to-table
+ * view's path. A plain table has no such path, so the base procedure does nothing.
+ */
+ protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) {
+ // do nothing here
+ }
+
@Override
protected boolean isRollbackSupported(final CreateTableState state) {
switch (state) {
@@ -259,10 +278,9 @@ public class CreateTableProcedure
}
}
- private void rollbackPreRelease(final ConfigNodeProcedureEnv env) {
+ protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) {
// A down DataNode must not block rollback if it is already provably self-fenced.
- final TUpdateTableReq req =
- SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null);
+ final TUpdateTableReq req = buildRollbackTableReq();
final boolean proceeded =
new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager()))
.propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets));
@@ -279,6 +297,10 @@ public class CreateTableProcedure
}
}
+ protected TUpdateTableReq buildRollbackTableReq() {
+ return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null);
+ }
+
@Override
protected CreateTableState getState(final int stateId) {
return CreateTableState.values()[stateId];
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java
index 54217212a11..00ec6655f7c 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java
@@ -254,7 +254,9 @@ public class DropTableProcedure extends AbstractAlterOrDropTableProcedure<DropTa
ProcedureMessages.COMMIT_RELEASE_DELETE_TABLE_WHEN_DROPPING_TABLE,
database,
tableName);
- commitRelease(env);
+ // A DataNode that misses this commit keeps the table in its pre-delete state, which only
+ // a later read of that table repairs. Retry until every reachable DataNode acks.
+ commitReleaseWithHA(env);
return Flow.NO_MORE_STATE;
default:
setFailure(new ProcedureException(ProcedureMessages.UNRECOGNIZED_DROPTABLESTATE + state));
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java
index 03088935ce6..6d8329e476a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java
@@ -94,7 +94,7 @@ public class RenameTableColumnProcedure
ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_RENAMING_COLUMN,
database,
tableName);
- commitRelease(env);
+ commitReleaseWithHA(env);
return Flow.NO_MORE_STATE;
default:
setFailure(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java
index 2bce5180505..c426558be78 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java
@@ -21,26 +21,37 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.exception.SemanticException;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema;
+import org.apache.iotdb.commons.schema.table.column.TagColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector;
import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
import org.apache.iotdb.confignode.procedure.state.ProcedureState;
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
+import javax.annotation.Nullable;
+
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.Optional;
+
+import static org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType.ALTER_TAG;
+import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA;
public class AddViewColumnProcedure extends AddTableColumnProcedure {
public AddViewColumnProcedure(final boolean isGeneratedByPipe) {
@@ -64,28 +75,56 @@ public class AddViewColumnProcedure extends AddTableColumnProcedure {
return;
}
- final Map<String, Set<FieldColumnSchema>> fields2Detect = new HashMap<>();
+ final Map<String, FieldColumnSchema> fields2Detect = new HashMap<>();
for (final TsTableColumnSchema schema : addedColumnList) {
if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) {
continue;
}
final String key = TreeViewSchema.getSourceName(schema);
- if (!fields2Detect.containsKey(key)) {
- fields2Detect.put(key, new HashSet<>());
+ if (fields2Detect.containsKey(key)) {
+ // The column check has already dropped the columns which refer to an already referenced
+ // source measurement, so the added columns must not share one.
+ throw new SemanticException(
+ String.format(
+ ManagerMessages
+ .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566,
+ key));
}
- fields2Detect.get(key).add((FieldColumnSchema) schema);
+ fields2Detect.put(key, (FieldColumnSchema) schema);
}
if (!fields2Detect.isEmpty()) {
final TSStatus status =
- new TreeDeviceViewFieldDetector(env.getConfigManager(), table, fields2Detect)
- .detectMissingFieldTypes();
+ new TreeDeviceViewFieldDetector(env.getConfigManager(), table)
+ .detectFieldTypesForColumnAddition(fields2Detect);
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
setFailure(new ProcedureException(new IoTDBException(status)));
}
}
}
+ @Override
+ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) {
+ final Optional<TreeToTableViewTrieOperationType> type = tree2TableViewOperationType();
+ if (type.isEmpty()) {
+ return super.buildCommitReleaseTableReq(oldName);
+ }
+ // Adding a tag column changes the tag count a device path is matched with.
+ final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType());
+ viewInfo.setTagNum(table.getTagNum());
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo);
+ }
+
+ private Optional<TreeToTableViewTrieOperationType> tree2TableViewOperationType() {
+ final Optional<String> propValue = table.getPropValue(AUTO_SYNC_SCHEMA);
+ if (propValue.isPresent() && Boolean.parseBoolean(propValue.get())) {
+ final boolean hasTagColumn =
+ addedColumnList.stream().anyMatch(c -> c instanceof TagColumnSchema);
+ return hasTagColumn ? Optional.of(ALTER_TAG) : Optional.empty();
+ }
+ return Optional.empty();
+ }
+
@Override
protected String getActionMessage() {
return "add view column";
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java
index f699a88a6ed..bf5551f457b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java
@@ -23,8 +23,10 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.schema.table.TableNodeStatus;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.iotdb.confignode.consensus.request.write.table.view.PreCreateTableViewPlan;
import org.apache.iotdb.confignode.exception.DatabaseNotExistsException;
import org.apache.iotdb.confignode.i18n.ProcedureMessages;
@@ -36,6 +38,8 @@ import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProced
import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState;
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.utils.Pair;
@@ -46,6 +50,7 @@ import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -116,13 +121,88 @@ public class CreateTableViewProcedure extends CreateTableProcedure {
}
}
final TSStatus status =
- new TreeDeviceViewFieldDetector(env.getConfigManager(), table, null)
- .detectMissingFieldTypes();
+ new TreeDeviceViewFieldDetector(env.getConfigManager(), table).detectFieldsForCreation();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
setFailure(new ProcedureException(new IoTDBException(status)));
}
}
+ @Override
+ protected TUpdateTableReq buildCommitReleaseTableReq() {
+ final boolean newAutoSync =
+ table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false);
+ if (newAutoSync) {
+ // Whether the view is a new one, or replace the old table, just add view
+ final TUpdateTree2TableView viewInfo =
+ new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType());
+ table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath);
+ viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table));
+ viewInfo.setTagNum(table.getTagNum());
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null, viewInfo);
+ }
+ // Creating, or replacing with the auto detection turned off: the view the old definition had
+ // registered has to go. drop a non-existing view is allowed.
+ final boolean oldAutoSync =
+ Objects.nonNull(oldView)
+ && oldView
+ .getPropValue(TsTable.AUTO_SYNC_SCHEMA)
+ .map(Boolean::parseBoolean)
+ .orElse(false);
+ if (oldAutoSync) {
+ return SchemaUtils.buildCommitUpdateTableReq(
+ database,
+ table.getTableName(),
+ null,
+ new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType()));
+ }
+ return super.buildCommitReleaseTableReq();
+ }
+
+ /**
+ * A rolled-back replace restores the view the old definition had. A DataNode which restarted
+ * while the replace was running does not hold it: the table such a DataNode pulls is only
+ * pre-created, and a pre-created table does not enter the index. Registering the old view again
+ * is idempotent, so a DataNode which never lost it is not affected.
+ */
+ @Override
+ protected TUpdateTableReq buildRollbackTableReq() {
+ final boolean oldAutoSync =
+ Objects.nonNull(oldView)
+ && oldView
+ .getPropValue(TsTable.AUTO_SYNC_SCHEMA)
+ .map(Boolean::parseBoolean)
+ .orElse(false);
+ if (!oldAutoSync) {
+ return super.buildRollbackTableReq();
+ }
+ final TUpdateTree2TableView viewInfo =
+ new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType());
+ oldView.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath);
+ viewInfo.setIsRestrict(TreeViewSchema.isRestrict(oldView));
+ viewInfo.setTagNum(oldView.getTagNum());
+ return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null, viewInfo);
+ }
+
+ @Override
+ protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) {
+ // The series which already existed under the path pattern were created before the DataNodes
+ // mounted the view, so they never triggered the addition of their column.
+ final List<TsTableColumnSchema> missingColumns =
+ TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation(
+ env.getConfigManager(), database, table);
+ if (missingColumns.isEmpty()) {
+ return;
+ }
+
+ addChildProcedure(
+ new AddViewColumnProcedure(
+ database,
+ table.getTableName(),
+ String.format("create_view_reconcile_%s.%s", database, table.getTableName()),
+ missingColumns,
+ false));
+ }
+
@Override
protected void preCreateTable(final ConfigNodeProcedureEnv env) {
final TSStatus status =
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java
index 163150c0a09..c365d1a3065 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java
@@ -19,9 +19,13 @@
package org.apache.iotdb.confignode.procedure.impl.schema.table.view;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
+import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
import org.apache.iotdb.confignode.procedure.state.schema.DropTableState;
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -39,6 +43,17 @@ public class DropViewProcedure extends DropTableProcedure {
super(database, tableName, queryId, isGeneratedByPipe);
}
+ /** A dropped view has to leave the tree-to-table view index of every DataNode. */
+ @Override
+ protected TUpdateTableReq buildCommitReleaseTableReq(final String oldName) {
+ // Unregistering a view needs nothing but its name, which the request already carries.
+ return SchemaUtils.buildCommitUpdateTableReq(
+ database,
+ table.getTableName(),
+ null,
+ new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType()));
+ }
+
/**
* A view holds no data, so the procedure goes straight from PRE_DELETE to DROP_TABLE. The view is
* removed from the ConfigNode once DROP_TABLE has completed; before that it can still be
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java
index 62f77fb3fd0..d6dec9689a3 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java
@@ -19,12 +19,20 @@
package org.apache.iotdb.confignode.procedure.impl.schema.table.view;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
+import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure;
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
+
+import javax.annotation.Nullable;
import java.io.DataOutputStream;
import java.io.IOException;
+import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA;
+
public class RenameViewProcedure extends RenameTableProcedure {
public RenameViewProcedure(final boolean isGeneratedByPipe) {
super(isGeneratedByPipe);
@@ -39,6 +47,17 @@ public class RenameViewProcedure extends RenameTableProcedure {
super(database, tableName, queryId, newName, isGeneratedByPipe);
}
+ @Override
+ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) {
+ if (!Boolean.parseBoolean(table.getPropValue(AUTO_SYNC_SCHEMA).orElse(null))) {
+ return super.buildCommitReleaseTableReq(oldName);
+ }
+ // A rename does not move the view, so the old name is all the DataNode needs.
+ final TUpdateTree2TableView viewInfo =
+ new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ALTER_RENAME.getOperationType());
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo);
+ }
+
@Override
protected String getActionMessage() {
return "rename view";
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java
index ce5b223cbb1..7919f98bc70 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java
@@ -19,12 +19,26 @@
package org.apache.iotdb.confignode.procedure.impl.schema.table.view;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
+import org.apache.iotdb.commons.schema.table.TreeViewSchema;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
+
+import javax.annotation.Nullable;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA;
public class SetViewPropertiesProcedure extends SetTablePropertiesProcedure {
public SetViewPropertiesProcedure(final boolean isGeneratedByPipe) {
@@ -40,6 +54,75 @@ public class SetViewPropertiesProcedure extends SetTablePropertiesProcedure {
super(database, tableName, queryId, properties, isGeneratedByPipe);
}
+ @Override
+ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) {
+ final Optional<TreeToTableViewTrieOperationType> type = tree2TableViewOperationType();
+ if (type.isEmpty()) {
+ return super.buildCommitReleaseTableReq(oldName);
+ }
+ final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType());
+ if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC == type.get()) {
+ // Turning the auto detection on mounts the view: the DataNode needs its whole definition.
+ table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath);
+ viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table));
+ viewInfo.setTagNum(table.getTagNum());
+ }
+ // turn down the auto_sync_schema
+ return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo);
+ }
+
+ @Override
+ protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) {
+ super.commitReleaseWithHA(env);
+ // Turning the auto detection on mounts the view, so the series which already exist under its
+ // path pattern need the same second detection as the ones of a newly created view.
+ if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC
+ == tree2TableViewOperationType().orElse(null)) {
+ reconcileTree2TableViewColumns(env);
+ }
+ }
+
+ /** Adds the columns which the view, just mounted by the commit, is still missing. */
+ private void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) {
+ final List<TsTableColumnSchema> missingColumns =
+ TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation(
+ env.getConfigManager(), database, table);
+ if (missingColumns.isEmpty()) {
+ return;
+ }
+ addChildProcedure(
+ new AddViewColumnProcedure(
+ database,
+ table.getTableName(),
+ String.format("set_view_properties_reconcile_%s.%s", database, table.getTableName()),
+ missingColumns,
+ false));
+ }
+
+ /**
+ * The effect of the new properties on the tree-to-table view index, or empty when they do not
+ * change it. Turning auto_sync_schema on registers the view, turning it off unregisters it, and
+ * any other property leaves the index alone.
+ */
+ private Optional<TreeToTableViewTrieOperationType> tree2TableViewOperationType() {
+ if (!updatedProperties.containsKey(AUTO_SYNC_SCHEMA)) {
+ return Optional.empty();
+ }
+ final TreeToTableViewTrieOperationType type =
+ indexEffectOf(updatedProperties.get(AUTO_SYNC_SCHEMA));
+ // An absent auto_sync_schema means false, so setting it to false is not a change.
+ return type == indexEffectOf(originalProperties.get(AUTO_SYNC_SCHEMA))
+ ? Optional.empty()
+ : Optional.of(type);
+ }
+
+ private static TreeToTableViewTrieOperationType indexEffectOf(
+ final @Nullable String autoSyncSchema) {
+ return Boolean.parseBoolean(autoSyncSchema)
+ ? TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC
+ : TreeToTableViewTrieOperationType.REMOVE;
+ }
+
@Override
protected String getActionMessage() {
return "set view properties";
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 1f89e39d448..489402cb11e 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -108,6 +108,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq;
@@ -1531,6 +1533,11 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.createTableView(req);
}
+ @Override
+ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) {
+ return configManager.autoSyncViewColumns(req);
+ }
+
@Override
public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp) {
return configManager.pushHeartbeat(dataNodeId, resp);
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
index 500d1d9c460..e5bbbdba230 100644
--- a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
+++ b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
@@ -91,6 +91,24 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
+ <!-- the error records of the tree-to-table view auto column addition, in a file of their own -->
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="VIEW_COLUMN_AUTO_SYNC_ERROR">
+ <file>${IOTDB_HOME}/logs/log_datanode_view_column_auto_sync_error.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_HOME}/logs/log-datanode-view-column-auto-sync-error-%d{yyyyMMdd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>ERROR</level>
+ <onMatch>ACCEPT</onMatch>
+ <onMismatch>DENY</onMismatch>
+ </filter>
+ </appender>
<appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
<Target>System.out</Target>
<encoder>
@@ -218,6 +236,9 @@
<logger level="OFF" name="io.moquette.broker.metrics.MQTTMessageLogger"/>
<logger level="info" name="org.apache.iotdb.db.service"/>
<logger level="info" name="org.apache.iotdb.db.conf"/>
+ <logger level="info" name="org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger">
+ <appender-ref ref="VIEW_COLUMN_AUTO_SYNC_ERROR"/>
+ </logger>
<logger level="info" name="org.apache.iotdb.db.cost.statistic">
<appender-ref ref="FILE_COST_MEASURE"/>
</logger>
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index b4271e3162f..f5ecb62b048 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -1544,7 +1544,8 @@ public final class DataNodeMiscMessages {
public static final String MESSAGE_CHANGEREGIONLEADER_SUCCESSFULLY_CHANGE_LEADER_REGIONGROUP_462797B4 = "[ChangeRegionLeader] Successfully change the leader of RegionGroup: ";
public static final String MESSAGE_MESSAGE_3501C7E6 = " to ";
public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards.";
- public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 = "The template is null when trying to activate template, may be the template is being unset.";
+ public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 =
+ "The template is null when trying to activate template, may be the template is being unset. The template ids are %s.";
public static final String MESSAGE_DOT_9D9B854A = ".";
public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 = "Error occurred while parsing SQL to physical plan: ";
public static final String MESSAGE_ERROR_OCCURRED_IN_READ_PROCESS_COLON_CD184195 = "Error occurred in read process: ";
@@ -1598,4 +1599,10 @@ public final class DataNodeMiscMessages {
LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 =
"mpp_data_exchange_max_payload_size_in_bytes (%d) is below the minimum allowed value %d, using %d";
+ public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB =
+ "Multiple types encountered when auto detecting type of measurement '%s', please check";
+
+ public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 =
+ "Failed to collect the devices using the extended template {}";
+
}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index d3282b78953..3f9abbc40ff 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -4064,6 +4064,8 @@ public final class DataNodeQueryMessages {
"Only InMemoryDeviceEntryDataSet supports get inline device entries";
public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 =
"Invalid LOAD consensus piece ref: path %s, offset %d, size %d";
+ public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 =
+ "auto_sync_schema is only supported by tree to table views";
public static final String MESSAGE_COPYTO_ARG_38343199 = "CopyTo-%s";
public static final String MESSAGE_TARGETFILE_ARG_EC484B24 = "TargetFile: %s";
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
index 06b0080d7a7..3e89ec3e4b0 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
@@ -567,6 +567,7 @@ public final class DataNodeSchemaMessages {
// ======================== DataNodeTableCache ========================
public static final String INIT_TABLE_CACHE_SUCCESS = "Init DataNodeTableCache successfully";
+ public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "Failed to register the tree to table view {}.{} during init: illegal path";
public static final String PRE_UPDATE_TABLE_SUCCESS = "Pre-update table {}.{} successfully";
public static final String PRE_RENAME_OLD_TABLE_SUCCESS =
"Pre-rename old table {}.{} successfully";
@@ -743,5 +744,25 @@ public final class DataNodeSchemaMessages {
public static final String EXCEPTION_COMMA_50AD1C01 = ", ";
public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "Skip rollback renaming old table {}.{} because it has been handled.";
public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "Skip commit-update table {}.{} because it has been handled.";
+ public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned {}.";
+ public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "Failed to add a column to the view {}.{} for operation {}, ConfigNode returned {}.";
+ public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned an empty response.";
+ public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "Failed to add columns to the tree to table views for operation {}, because ConfigNode is not reachable.";
+ public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected.";
+ public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "Tree to table view {}.{} is not registered";
+ public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view {}.{}";
+ public static final String
+ LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 =
+ "Registered the tree to table view {}.{} at {} with restrict {} and num of tags {}";
+ public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A =
+ "Unregistered the tree to table view {}.{}";
+ public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 =
+ "Renamed the tree to table view {}.{} from {}";
+ public static final String
+ LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 =
+ "Updated the properties of the tree to table view {}.{} to {}";
+ public static final String
+ LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 =
+ "The metadata lease is fenced while adding view columns, giving up this round; the measurements are {}";
}
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index 0139e97dcf6..26e6a16a555 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -1524,8 +1524,8 @@ public final class DataNodeMiscMessages {
"[ChangeRegionLeader] 成功切换 RegionGroup 的 leader:";
public static final String MESSAGE_MESSAGE_3501C7E6 = " 到 ";
public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "\"executeFastLastDataQueryForOnePrefixPath\" 不支持通配符。";
- public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 =
- "尝试激活模板时模板为空,可能正在取消设置该模板。";
+ public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 =
+ "尝试激活模板时模板为空,可能正在取消设置该模板。模板 ID 为 %s。";
public static final String MESSAGE_DOT_9D9B854A = ".";
public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 =
"将 SQL 解析为物理计划时发生错误:";
@@ -1581,4 +1581,10 @@ public final class DataNodeMiscMessages {
LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 =
"mpp_data_exchange_max_payload_size_in_bytes(%d)低于允许的最小值 %d,将使用 %d";
+ public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB =
+ "自动检测测点 %s 的类型时遇到多种类型,请检查";
+
+ public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 =
+ "收集使用扩展模板 {} 的设备失败";
+
}
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 6a50a255cf6..12cb3b07a05 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -4820,6 +4820,8 @@ public final class DataNodeQueryMessages {
"只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目";
public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 =
"无效的 LOAD 共识分片引用:路径 %s,偏移 %d,大小 %d";
+ public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 =
+ "auto_sync_schema 仅支持树转表视图";
public static final String MESSAGE_COPYTO_ARG_38343199 = "CopyTo-%s";
public static final String MESSAGE_TARGETFILE_ARG_EC484B24 = "目标文件:%s";
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
index a932bc061cd..366ba2cfbd4 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java
@@ -568,6 +568,7 @@ public final class DataNodeSchemaMessages {
// ======================== DataNodeTableCache 相关消息 ========================
public static final String INIT_TABLE_CACHE_SUCCESS = "DataNodeTableCache 初始化成功";
+ public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "初始化时注册树转表视图 {}.{} 失败:路径非法";
public static final String PRE_UPDATE_TABLE_SUCCESS = "预更新表 {}.{} 成功";
public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "预重命名旧表 {}.{} 成功";
public static final String ROLLBACK_UPDATE_TABLE_SUCCESS = "回滚更新表 {}.{} 成功";
@@ -731,5 +732,25 @@ public final class DataNodeSchemaMessages {
public static final String EXCEPTION_COMMA_50AD1C01 = ", ";
public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "跳过回滚重命名旧表 {}.{},因为该操作已被处理。";
public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "跳过提交更新表 {}.{},因为该操作已被处理。";
+ public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "为操作 {} 的树转表视图加列失败,ConfigNode 返回 {}。";
+ public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "为视图 {}.{} 加列失败(操作 {}),ConfigNode 返回 {}。";
+ public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "为操作 {} 的树转表视图加列失败,ConfigNode 返回了空响应。";
+ public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "为操作 {} 的树转表视图加列失败,因为无法访问 ConfigNode。";
+ public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "计划节点 {} 触发树转表视图加列失败,本次 schema 写入不受影响。";
+ public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "树转表视图 {}.{} 未注册";
+ public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 {}.{} 的索引失败";
+ public static final String
+ LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 =
+ "注册树转表视图 {}.{},路径 {},restrict {},tag 数 {}";
+ public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A =
+ "注销树转表视图 {}.{}";
+ public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 =
+ "树转表视图 {}.{} 由 {} 重命名而来";
+ public static final String
+ LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 =
+ "树转表视图 {}.{} 的属性更新为 {}";
+ public static final String
+ LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 =
+ "自动扩列时元数据租约已隔离,本次放弃;涉及的测点为 {}";
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
index a77b0895e4c..47b353aefa8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
@@ -79,6 +79,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateTimeSeries
import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory;
import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl;
import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl;
+import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache;
+import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger;
import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -94,9 +96,14 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
/** Schema write {@link PlanNode} visitor */
public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegion> {
@@ -111,6 +118,10 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
logMetaDataException(String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e);
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
+ final MeasurementPath path = node.getPath();
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurement =
+ toSingleMeasurement(path.getDevicePath(), path.getMeasurement(), node.getDataType());
+ syncViewColumns(node, createdMeasurement);
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully");
}
@@ -138,6 +149,11 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
logMetaDataException(String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e);
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
+
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements =
+ Collections.singletonMap(
+ node.getDevicePath(), toMeasurements(node.getMeasurements(), node.getDataTypes()));
+ syncViewColumns(node, createdMeasurements);
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully");
}
@@ -145,6 +161,7 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
public TSStatus visitCreateMultiTimeSeries(
final CreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) {
final Map<PartialPath, MeasurementGroup> measurementGroupMap = node.getMeasurementGroupMap();
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements = new HashMap<>();
final List<TSStatus> failingStatus = new ArrayList<>();
PartialPath devicePath;
MeasurementGroup measurementGroup;
@@ -160,6 +177,11 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
transformToCreateTimeSeriesPlan(devicePath, measurementGroup, i);
((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(node.isGeneratedByPipe());
schemaRegion.createTimeSeries(createTimeSeriesPlan, -1);
+ recordCreatedMeasurement(
+ createdMeasurements,
+ devicePath,
+ measurementGroup.getMeasurements().get(i),
+ measurementGroup.getDataTypes().get(i));
} catch (final MetadataException e) {
logMetaDataException(
String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e);
@@ -168,6 +190,10 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
}
}
+ // The measurements which were created need their columns even when another one of the same
+ // statement failed: no later write of them triggers this addition again.
+ syncViewColumns(node, createdMeasurements);
+
if (!failingStatus.isEmpty()) {
return RpcUtils.getStatus(failingStatus);
}
@@ -194,9 +220,11 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
@Override
public TSStatus visitInternalCreateTimeSeries(
final InternalCreateTimeSeriesNode node, final ISchemaRegion schemaRegion) {
+
final PartialPath devicePath = node.getDevicePath();
final MeasurementGroup measurementGroup = node.getMeasurementGroup();
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements = new HashMap<>();
final List<TSStatus> existingTimeSeriesAndAlignmentMismatch = new ArrayList<>();
final List<TSStatus> failingStatus = new ArrayList<>();
@@ -207,7 +235,8 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
schemaRegion,
existingTimeSeriesAndAlignmentMismatch,
failingStatus,
- node.isGeneratedByPipe());
+ node.isGeneratedByPipe(),
+ createdMeasurements);
} else {
executeInternalCreateTimeSeries(
devicePath,
@@ -215,9 +244,12 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
schemaRegion,
existingTimeSeriesAndAlignmentMismatch,
failingStatus,
- node.isGeneratedByPipe());
+ node.isGeneratedByPipe(),
+ createdMeasurements);
}
+ syncViewColumns(node, createdMeasurements);
+
if (!failingStatus.isEmpty()) {
return RpcUtils.getStatus(failingStatus);
}
@@ -232,6 +264,8 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
@Override
public TSStatus visitInternalCreateMultiTimeSeries(
final InternalCreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) {
+
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements = new HashMap<>();
PartialPath devicePath;
MeasurementGroup measurementGroup;
@@ -249,7 +283,8 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
schemaRegion,
existingTimeSeriesAndAlignmentMismatch,
failingStatus,
- node.isGeneratedByPipe());
+ node.isGeneratedByPipe(),
+ createdMeasurements);
} else {
executeInternalCreateTimeSeries(
devicePath,
@@ -257,10 +292,13 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
schemaRegion,
existingTimeSeriesAndAlignmentMismatch,
failingStatus,
- node.isGeneratedByPipe());
+ node.isGeneratedByPipe(),
+ createdMeasurements);
}
}
+ syncViewColumns(node, createdMeasurements);
+
if (!failingStatus.isEmpty()) {
return RpcUtils.getStatus(failingStatus);
}
@@ -278,7 +316,8 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
final ISchemaRegion schemaRegion,
final List<TSStatus> existingTimeSeriesAndAlignmentMismatch,
final List<TSStatus> failingStatus,
- final boolean withMerge) {
+ final boolean withMerge,
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements) {
final int size = measurementGroup.getMeasurements().size();
// todo implement batch creation of one device in SchemaRegion
boolean alignedIsSet = false;
@@ -292,6 +331,13 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
// Thus the original ones are not altered
((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(withMerge);
schemaRegion.createTimeSeries(createTimeSeriesPlan, -1);
+ // The alignment mismatch below does not undo the creation, so the measurement is created
+ // either way.
+ recordCreatedMeasurement(
+ createdMeasurements,
+ devicePath,
+ measurementGroup.getMeasurements().get(i),
+ measurementGroup.getDataTypes().get(i));
if (((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).getAligned().get() && !alignedIsSet) {
existingTimeSeriesAndAlignmentMismatch.add(
new TSStatus(TSStatusCode.ALIGNED_TIMESERIES_ERROR.getStatusCode())
@@ -316,7 +362,8 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
final ISchemaRegion schemaRegion,
final List<TSStatus> existingTimeSeriesAndAlignmentMismatch,
final List<TSStatus> failingStatus,
- final boolean withMerge) {
+ final boolean withMerge,
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements) {
final List<String> measurementList = measurementGroup.getMeasurements();
final List<TSDataType> dataTypeList = measurementGroup.getDataTypes();
final List<TSEncoding> encodingList = measurementGroup.getEncodings();
@@ -344,8 +391,19 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
boolean shouldRetry = true;
while (shouldRetry) {
+ // The retry trims the plan, so the measurements of this attempt are the ones the plan holds
+ // now. They are captured before the call, as the call itself trims the measurements it merges
+ // instead of creating.
+ final List<String> attemptedMeasurements =
+ new ArrayList<>(createAlignedTimeSeriesPlan.getMeasurements());
+ final List<TSDataType> attemptedDataTypes =
+ new ArrayList<>(createAlignedTimeSeriesPlan.getDataTypes());
try {
schemaRegion.createAlignedTimeSeries(createAlignedTimeSeriesPlan);
+ // An aligned batch is validated as a whole before it is created, so a call which returns
+ // without an exception has created every measurement it was given.
+ recordCreatedMeasurements(
+ createdMeasurements, devicePath, attemptedMeasurements, attemptedDataTypes);
shouldRetry = false;
} catch (final MeasurementAlreadyExistException e) {
// The existence check will be executed before truly creation
@@ -411,6 +469,81 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
}
}
+ /**
+ * Adds to the tree-to-table views the columns the measurements a schema write has created need.
+ *
+ * <p>It is a best-effort side effect of the write which triggers it: every failure is logged here
+ * and none of them is propagated, so that the write operation is never affected.
+ *
+ * <p>A plan which a pipe pushed is skipped: the cluster the measurements come from has already
+ * added their columns, and those columns reach this cluster with the view schema itself, so
+ * asking ConfigNode for them again can only be rejected as a duplicate.
+ *
+ * @param node the plan which triggers the addition
+ * @param createdMeasurements the measurements this operation really created; only they get a
+ * column, so that a creation which failed leaves no column behind
+ */
+ private void syncViewColumns(
+ final WritePlanNode node,
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements) {
+ if (node.isGeneratedByPipe()) {
+ return;
+ }
+ final String operationId = node.getPlanNodeId().getId();
+ try {
+ if (TreeToTableViewCache.getInstance().hasView()) {
+ ViewColumnAutoSyncTrigger.sync(createdMeasurements, operationId);
+ }
+ } catch (final Throwable e) {
+ // Logged with the logger of the feature, so that every failure of the auto column addition,
+ // including the ones the trigger itself does not see, reaches its dedicated log file.
+ ViewColumnAutoSyncTrigger.LOGGER.error(
+ DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F, operationId, e);
+ }
+ }
+
+ private static void recordCreatedMeasurement(
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements,
+ final PartialPath devicePath,
+ final String measurement,
+ final TSDataType dataType) {
+ createdMeasurements
+ .computeIfAbsent(devicePath, key -> new ArrayList<>())
+ .add(new Pair<>(measurement, dataType));
+ }
+
+ private static void recordCreatedMeasurements(
+ final Map<PartialPath, List<Pair<String, TSDataType>>> createdMeasurements,
+ final PartialPath devicePath,
+ final List<String> measurements,
+ final List<TSDataType> dataTypes) {
+ for (int i = 0; i < measurements.size(); i++) {
+ recordCreatedMeasurement(
+ createdMeasurements, devicePath, measurements.get(i), dataTypes.get(i));
+ }
+ }
+
+ private static Map<PartialPath, List<Pair<String, TSDataType>>> toSingleMeasurement(
+ final PartialPath devicePath, final String measurement, final TSDataType dataType) {
+ return Collections.singletonMap(
+ devicePath, Collections.singletonList(new Pair<>(measurement, dataType)));
+ }
+
+ private static List<Pair<String, TSDataType>> toMeasurements(final Template template) {
+ return template.getSchemaMap().values().stream()
+ .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType()))
+ .toList();
+ }
+
+ private static List<Pair<String, TSDataType>> toMeasurements(
+ final List<String> measurements, final List<TSDataType> dataTypes) {
+ final List<Pair<String, TSDataType>> pairs = new ArrayList<>(measurements.size());
+ for (int i = 0; i < measurements.size(); i++) {
+ pairs.add(new Pair<>(measurements.get(i), dataTypes.get(i)));
+ }
+ return pairs;
+ }
+
@Override
public TSStatus visitAlterTimeSeries(
final AlterTimeSeriesNode node, final ISchemaRegion schemaRegion) {
@@ -465,43 +598,49 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
@Override
public TSStatus visitActivateTemplate(
final ActivateTemplateNode node, final ISchemaRegion schemaRegion) {
+ final Template template =
+ ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId());
+ if (Objects.isNull(template)) {
+ return getUndefinedTemplateStatus(Collections.singleton(node.getTemplateId()));
+ }
try {
- final Template template =
- ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId());
- if (Objects.isNull(template)) {
- return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode())
- .setMessage(
- DataNodeMiscMessages
- .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779);
- }
schemaRegion.activateSchemaTemplate(node, template);
- return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
} catch (final MetadataException e) {
logMetaDataException(e);
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
+ final PartialPath devicePath = node.getActivatePath();
+ final Map<PartialPath, List<Pair<String, TSDataType>>> activatedMeasurements =
+ Collections.singletonMap(devicePath, toMeasurements(template));
+ syncViewColumns(node, activatedMeasurements);
+ return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
}
@Override
public TSStatus visitBatchActivateTemplate(
final BatchActivateTemplateNode node, final ISchemaRegion schemaRegion) {
+ final Map<Integer, Template> templateMap = new HashMap<>();
+ final Set<Integer> missingTemplateIds = new HashSet<>();
+ collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds);
+
+ final Map<PartialPath, List<Pair<String, TSDataType>>> activatedMeasurements = new HashMap<>();
final List<TSStatus> statusList = new ArrayList<>();
final List<PartialPath> alreadyActivatedDeviceList = new ArrayList<>();
for (final Map.Entry<PartialPath, Pair<Integer, Integer>> entry :
node.getTemplateActivationMap().entrySet()) {
- final Template template =
- ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left);
+ final Template template = templateMap.get(entry.getValue().left);
if (Objects.isNull(template)) {
- return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode())
- .setMessage(
- DataNodeMiscMessages
- .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779);
+ // its id is in missingTemplateIds, which is reported at the end
+ continue;
}
try {
schemaRegion.activateSchemaTemplate(
SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan(
entry.getKey(), entry.getValue().right, entry.getValue().left),
template);
+ // A device which already carries the template is reported below and gets no column here:
+ // its measurements were already covered when the template was activated on it.
+ activatedMeasurements.put(entry.getKey(), toMeasurements(template));
} catch (final MetadataException e) {
if (e.getErrorCode() == TSStatusCode.TEMPLATE_IS_IN_USE.getStatusCode()) {
alreadyActivatedDeviceList.add(entry.getKey());
@@ -511,34 +650,44 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
}
}
}
+ syncViewColumns(node, activatedMeasurements);
if (!alreadyActivatedDeviceList.isEmpty()) {
final TemplateIsInUseException e =
new TemplateIsInUseException(alreadyActivatedDeviceList.toString());
logMetaDataException(e);
statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage()));
}
+ if (!missingTemplateIds.isEmpty()) {
+ statusList.add(getUndefinedTemplateStatus(missingTemplateIds));
+ }
return statusList.isEmpty() ? RpcUtils.SUCCESS_STATUS : RpcUtils.getStatus(statusList);
}
@Override
public TSStatus visitInternalBatchActivateTemplate(
final InternalBatchActivateTemplateNode node, final ISchemaRegion schemaRegion) {
+ final Map<Integer, Template> templateMap = new HashMap<>();
+ final Set<Integer> missingTemplateIds = new HashSet<>();
+ collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds);
+
+ final Map<PartialPath, List<Pair<String, TSDataType>>> activatedMeasurements = new HashMap<>();
+
for (final Map.Entry<PartialPath, Pair<Integer, Integer>> entry :
node.getTemplateActivationMap().entrySet()) {
- final Template template =
- ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left);
+ final Template template = templateMap.get(entry.getValue().left);
if (Objects.isNull(template)) {
- return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode())
- .setMessage(
- DataNodeMiscMessages
- .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779);
+ // its id is in missingTemplateIds, which is reported at the end
+ continue;
}
try {
schemaRegion.activateSchemaTemplate(
SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan(
entry.getKey(), entry.getValue().right, entry.getValue().left),
template);
+ activatedMeasurements.put(entry.getKey(), toMeasurements(template));
} catch (final TemplateIsInUseException e) {
+ // The measurements of a device which already carries the template were already covered when
+ // the template was activated on it.
logger.info(
String.format(
DataNodeSchemaMessages.DEVICE_TEMPLATE_ALREADY_ACTIVATED_ON_PATH_FMT,
@@ -548,9 +697,44 @@ public class SchemaExecutionVisitor implements PlanVisitor<TSStatus, ISchemaRegi
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
}
+ syncViewColumns(node, activatedMeasurements);
+ if (!missingTemplateIds.isEmpty()) {
+ return getUndefinedTemplateStatus(missingTemplateIds);
+ }
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
}
+ private static void collectTemplates(
+ final Map<PartialPath, Pair<Integer, Integer>> templateActivationMap,
+ final Map<Integer, Template> templateMap,
+ final Set<Integer> missingTemplateIds) {
+ final ClusterTemplateManager templateManager = ClusterTemplateManager.getInstance();
+ for (final Pair<Integer, Integer> activation : templateActivationMap.values()) {
+ final int templateId = activation.left;
+ if (templateMap.containsKey(templateId) || missingTemplateIds.contains(templateId)) {
+ // already resolved for another path of this plan
+ continue;
+ }
+ final Template template = templateManager.getTemplate(templateId);
+ if (Objects.isNull(template)) {
+ missingTemplateIds.add(templateId);
+ } else {
+ templateMap.put(templateId, template);
+ }
+ }
+ }
+
+ private static TSStatus getUndefinedTemplateStatus(final Set<Integer> missingTemplateIds) {
+ return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode())
+ .setMessage(
+ String.format(
+ DataNodeMiscMessages
+ .MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72,
+ missingTemplateIds.stream()
+ .map(String::valueOf)
+ .collect(Collectors.joining(", "))));
+ }
+
@Override
public TSStatus visitConstructSchemaBlackList(
final ConstructSchemaBlackListNode node, final ISchemaRegion schemaRegion) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index 9f1e90fda10..02d6c51ef2d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@ -63,6 +63,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq;
@@ -1315,6 +1317,13 @@ public class ConfigNodeClient extends AbstractConfigNodeClient<ConfigNodeClient>
() -> client.createTableView(req), status -> !updateConfigNodeLeader(status));
}
+ @Override
+ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req)
+ throws TException {
+ return executeRemoteCallWithRetry(
+ () -> client.autoSyncViewColumns(req), status -> !updateConfigNodeLeader(status.status));
+ }
+
@Override
public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp)
throws TException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index e9b26334f66..5d3bce4e4c3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -192,9 +192,13 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager;
import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
+import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory;
+import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo;
import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo;
import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache;
+import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger;
import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager;
import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType;
import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil;
@@ -406,6 +410,7 @@ import java.util.stream.Stream;
import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE;
+import static org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger.EXTEND_DEVICE_TEMPLATE_ID_PREFIX;
import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS;
import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException;
import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException;
@@ -1959,14 +1964,16 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
.rollbackUpdateTable(
ReadWriteIOUtils.readString(req.tableInfo),
ReadWriteIOUtils.readString(req.tableInfo),
- req.oldName);
+ req.oldName,
+ req.updateTree2TableViewInfo);
break;
case COMMIT_UPDATE_TABLE:
DataNodeTableCache.getInstance()
.commitUpdateTable(
ReadWriteIOUtils.readString(req.tableInfo),
ReadWriteIOUtils.readString(req.tableInfo),
- req.oldName);
+ req.oldName,
+ req.updateTree2TableViewInfo);
break;
default:
LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TABLE, req.type);
@@ -2149,13 +2156,15 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
for (int i = 0; i < req.getPrefixPattern().size() - 1; ++i) {
nodes[i] = req.getPrefixPattern().get(i);
}
+ final ExtendedPartialPath pathPattern =
+ new ExtendedPartialPath(nodes, req.isRestrict());
final ISchemaSource<ITimeSeriesSchemaInfo> schemaSource =
SchemaSourceFactory.getTimeSeriesSchemaCountSource(
- new ExtendedPartialPath(nodes, req.isRestrict()),
+ pathPattern,
false,
// Does not support logical view currently
SchemaFilterFactory.createViewTypeFilter(ViewType.BASE),
- null,
+ ClusterTemplateManager.getInstance().checkAllRelatedTemplate(pathPattern),
SchemaConstant.ALL_MATCH_SCOPE,
true,
true);
@@ -2173,11 +2182,13 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
}
if (!updateMap.containsKey(measurementName)) {
updateMap.put(measurementName, schema.getTypeInByte());
- } else if (schema.getTypeInByte() != updateMap.get(measurementName)) {
+ } else if (schema.getTypeInByte() != updateMap.get(measurementName)
+ && !req.isTolerateTypeMismatch()) {
return RpcUtils.getStatus(
TSStatusCode.DATA_TYPE_MISMATCH,
String.format(
- "Multiple types encountered when auto detecting type of measurement '%s', please check",
+ DataNodeMiscMessages
+ .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB,
measurementName));
}
}
@@ -2902,6 +2913,23 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
SchemaEngine.getInstance()
.updateSubtreeMeasurementCountForTemplate(newTemplate.getId(), delta);
}
+ if (TreeToTableViewCache.getInstance().hasView()) {
+ // ViewColumnAutoSync.sync, which collects the views from the index and asks CN to add the
+ // columns.
+ try {
+ final Map<PartialPath, List<Pair<String, TSDataType>>> device2Measurements =
+ collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate);
+ ViewColumnAutoSyncTrigger.sync(
+ device2Measurements, EXTEND_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId());
+ } catch (final Exception e) {
+ ViewColumnAutoSyncTrigger.LOGGER.warn(
+ DataNodeMiscMessages
+ .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554,
+ newTemplate.getName(),
+ e);
+ }
+ }
+
break;
default:
LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TEMPLATE, req.type);
@@ -2910,6 +2938,68 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
}
+ /**
+ * The devices which activated the template, each with the measurements the extension added to it:
+ * the tree-to-table views covering these devices need them as columns.
+ *
+ * <p>The measurements come from comparing the new template with the old one. The devices come
+ * from the schema regions, where every device which uses a template records its id, see {@code
+ * EntityTraverser}; the template manager only knows where a template is mounted, so it cannot
+ * tell which devices use it.
+ */
+ private Map<PartialPath, List<Pair<String, TSDataType>>> collectDevicesUsingExtendedTemplate(
+ final Template oldTemplate, final Template newTemplate) throws Exception {
+ final List<Pair<String, TSDataType>> extendedMeasurements =
+ newTemplate.getSchemaMap().values().stream()
+ .filter(
+ schema ->
+ Objects.isNull(oldTemplate)
+ || !oldTemplate.getSchemaMap().containsKey(schema.getMeasurementName()))
+ .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType()))
+ .toList();
+ final Pair<Template, List<PartialPath>> templateSetInfo =
+ ClusterTemplateManager.getInstance().getAllPathsSetTemplate(newTemplate.getName());
+ final List<PartialPath> setPaths =
+ Objects.isNull(templateSetInfo) ? null : templateSetInfo.getRight();
+ if (extendedMeasurements.isEmpty() || Objects.isNull(setPaths)) {
+ return Collections.emptyMap();
+ }
+
+ final Map<PartialPath, List<Pair<String, TSDataType>>> device2Measurements = new HashMap<>();
+ Collection<ISchemaRegion> allSchemaRegions = SchemaEngine.getInstance().getAllSchemaRegions();
+ for (final PartialPath setPath : setPaths) {
+ // A device which is exactly the mount path has the template too, but "setPath.**" only
+ // matches its descendants, so the mount path itself is read with a pattern of its own.
+ final List<PartialPath> devicePatterns =
+ Arrays.asList(setPath, setPath.concatNode(MULTI_LEVEL_PATH_WILDCARD));
+ for (final ISchemaRegion schemaRegion : allSchemaRegions) {
+ if (!setPath.startsWithOrPrefixOf(
+ PathUtils.splitPathToDetachedNodes(schemaRegion.getDatabaseFullPath()))) {
+ continue;
+ }
+ for (final PartialPath devicePattern : devicePatterns) {
+ try (final ISchemaReader<IDeviceSchemaInfo> deviceReader =
+ schemaRegion.getDeviceReader(
+ SchemaRegionReadPlanFactory.getShowDevicesPlan(
+ devicePattern,
+ 0,
+ 0,
+ false,
+ newTemplate.getId(),
+ SchemaConstant.ALL_MATCH_SCOPE))) {
+ while (deviceReader.hasNext()) {
+ device2Measurements
+ .computeIfAbsent(
+ deviceReader.next().getPartialPath(), device -> new ArrayList<>())
+ .addAll(extendedMeasurements);
+ }
+ }
+ }
+ }
+ }
+ return device2Measurements;
+ }
+
@Override
public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) {
TSStatus consensusStatus = waitForConsensusStarted();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
index 033ba02836c..49ddfda6fac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
@@ -304,6 +304,7 @@ import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROP
import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES;
import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME;
import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY;
+import static org.apache.iotdb.db.i18n.DataNodeQueryMessages.EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215;
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY;
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY;
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY;
@@ -633,8 +634,7 @@ public class TableConfigTaskVisitor implements AstVisitor<IConfigTask, MPPQueryC
context.getSession().getUserName(), new QualifiedObjectName(database, tableName), context);
final TsTable table = new TsTable(tableName);
-
- table.setProps(convertPropertiesToMap(node.getProperties(), false));
+ table.setProps(convertPropertiesToMap(node.getProperties(), false, node instanceof CreateView));
if (Objects.nonNull(node.getComment())) {
table.addProp(TsTable.COMMENT_KEY, node.getComment());
}
@@ -902,7 +902,7 @@ public class TableConfigTaskVisitor implements AstVisitor<IConfigTask, MPPQueryC
return new AlterTableSetPropertiesTask(
database,
tableName,
- convertPropertiesToMap(node.getProperties(), true),
+ convertPropertiesToMap(node.getProperties(), true, isTreeView),
context.getQueryId().getId(),
node.ifExists(),
isTreeView);
@@ -985,8 +985,20 @@ public class TableConfigTaskVisitor implements AstVisitor<IConfigTask, MPPQueryC
}
}
+ /**
+ * Converts the property assignments of a CREATE TABLE/VIEW or a SET PROPERTIES statement into the
+ * props of the TsTable.
+ *
+ * @param propertyList the property assignments to convert
+ * @param serializeDefault whether a property assigned {@code DEFAULT} shall be kept in the result
+ * as a null value, which resets the property, instead of being dropped
+ * @param allowAutoSyncSchema whether the statement targets a tree to table view, the only kind of
+ * table whose schema can be synchronized from a tree path prefix
+ */
private Map<String, String> convertPropertiesToMap(
- final List<Property> propertyList, final boolean serializeDefault) {
+ final List<Property> propertyList,
+ final boolean serializeDefault,
+ final boolean allowAutoSyncSchema) {
final Map<String, String> map = new HashMap<>();
final Set<String> deduplicate = new HashSet<>();
for (final Property property : propertyList) {
@@ -995,6 +1007,11 @@ public class TableConfigTaskVisitor implements AstVisitor<IConfigTask, MPPQueryC
throw new SemanticException(DataNodeQueryMessages.DUPLICATED_PROPERTY + key);
}
if (TABLE_ALLOWED_PROPERTIES.contains(key)) {
+ if (TsTable.AUTO_SYNC_SCHEMA.equals(key) && !allowAutoSyncSchema) {
+ // A property assigned DEFAULT is dropped or reset as well, so the check covers it too
+ throw new SemanticException(
+ EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215);
+ }
if (!property.isSetToDefault()) {
final Expression value = property.getNonDefaultValue();
switch (key) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java
index a616282e7c4..7d2905e9017 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java
@@ -147,7 +147,12 @@ public class ShowCreateViewTask extends AbstractTableTask {
if (ttlString.equals(TTL_INFINITE)) {
ttlString = "'" + ttlString + "'";
}
- builder.append(" WITH (ttl=").append(ttlString).append(")");
+ builder.append(" WITH (ttl=").append(ttlString);
+ // An absent auto_sync_schema means false, so it only has to be given back when it is stored
+ table
+ .getPropValue(TsTable.AUTO_SYNC_SCHEMA)
+ .ifPresent(autoSyncSchema -> builder.append(", auto_sync_schema=").append(autoSyncSchema));
+ builder.append(")");
builder.append(" AS ");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java
index 7f7365a1550..5f3c7357c88 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.i18n.DataNodeSchemaMessages;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +109,8 @@ public class MetadataLeaseManager {
() -> ClusterPartitionFetcher.getInstance().invalidAllCache(),
() -> DataNodeTableCache.getInstance().invalidateAll(),
() -> TreeDeviceSchemaCacheManager.getInstance().cleanUp(),
- AuthorityChecker::invalidateAllCache);
+ AuthorityChecker::invalidateAllCache,
+ () -> TreeToTableViewCache.getInstance().clear());
}
private static List<MetadataAction> defaultPullMetaList() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java
new file mode 100644
index 00000000000..e9eb73ae802
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.schemaengine.schemaregion.view;
+
+import java.util.Objects;
+
+/** A view identity and its matching metadata. Equality depends only on database and table names. */
+public final class TreeToTableViewReference {
+
+ public static final String NUM_OF_TAGS = "num_of_tags";
+
+ private final String databaseName;
+
+ private final String tableName;
+
+ private final boolean restrict;
+
+ private final int numOfTags;
+
+ public TreeToTableViewReference(
+ String databaseName, String tableName, boolean restrict, int numOfTags) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.restrict = restrict;
+ this.numOfTags = numOfTags;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public boolean isRestrict() {
+ return restrict;
+ }
+
+ public int getNumOfTags() {
+ return numOfTags;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TreeToTableViewReference)) {
+ return false;
+ }
+ TreeToTableViewReference that = (TreeToTableViewReference) obj;
+ return Objects.equals(databaseName, that.databaseName)
+ && Objects.equals(tableName, that.tableName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(databaseName, tableName);
+ }
+
+ @Override
+ public String toString() {
+ return "TreeToTableViewReference{"
+ + "databaseName='"
+ + databaseName
+ + '\''
+ + ", tableName='"
+ + tableName
+ + '\''
+ + ", restrict="
+ + restrict
+ + ", numOfTags="
+ + numOfTags
+ + '}';
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
index 3db7af6f89a..1f10cb7f4b5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
@@ -23,13 +23,16 @@ import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.consensus.ConfigRegionId;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.i18n.CommonMessages;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.NonCommittableTsTable;
import org.apache.iotdb.commons.schema.table.PreDeleteTsTable;
import org.apache.iotdb.commons.schema.table.TableNodeStatus;
+import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
@@ -44,6 +47,8 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager;
+import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.thrift.TException;
@@ -55,11 +60,13 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -69,6 +76,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.stream.Collectors;
+import static org.apache.iotdb.commons.schema.table.TreeViewSchema.TREE_PATH_PATTERN;
import static org.apache.iotdb.db.i18n.DataNodeSchemaMessages.FAILED_TO_REFRESH_CACHE_FROM_CN;
/** It contains all tables' latest column schema */
@@ -77,6 +85,7 @@ public class DataNodeTableCache implements ITableCache {
private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeTableCache.class);
private static final IClientManager<ConfigRegionId, ConfigNodeClient> CONFIG_NODE_CLIENT_MANAGER =
ConfigNodeClientManager.getInstance();
+ private final ITreeToTableViewCache treeToTableViewCache = TreeToTableViewCache.getInstance();
/** Instance-specific version counter for optimistic locking mechanisms. */
private final AtomicLong instanceVersion = new AtomicLong(0);
@@ -142,12 +151,39 @@ public class DataNodeTableCache implements ITableCache {
table -> new Pair<>(table, 0L),
(v1, v2) -> v2,
ConcurrentHashMap::new))));
+ treeToTableViewCache.init(collectInitialViews(usingMap));
LOGGER.info(DataNodeSchemaMessages.INIT_TABLE_CACHE_SUCCESS);
} finally {
readWriteLock.writeLock().unlock();
}
}
+ private Map<String, List<TsTable>> collectInitialViews(
+ final Map<String, List<TsTable>> usingMap) {
+ final Map<String, List<TsTable>> initialViews = new HashMap<>();
+ usingMap.forEach(
+ (key, value) -> {
+ final List<TsTable> views =
+ value.stream().filter(DataNodeTableCache::isAutoSyncView).toList();
+ if (!views.isEmpty()) {
+ initialViews.put(PathUtils.unQualifyDatabaseName(key), new ArrayList<>(views));
+ }
+ });
+ return initialViews;
+ }
+
+ /**
+ * Whether the table belongs in the tree-to-table view index: it is a tree-to-table view which
+ * auto-syncs its schema. A NonCommittableTsTable and a PreDeleteTsTable are placeholders which
+ * carry no definition at all, so they are never one.
+ */
+ private static boolean isAutoSyncView(final TsTable table) {
+ final Optional<String> autoSyncSchemaValue = table.getPropValue(TsTable.AUTO_SYNC_SCHEMA);
+ return autoSyncSchemaValue.isPresent()
+ && Boolean.parseBoolean(autoSyncSchemaValue.get())
+ && table.getPropValue(TREE_PATH_PATTERN).isPresent();
+ }
+
// No need to acquire a lock here; reloadTableCacheAfterLeaseRecovery is within a critical section
// protected by metadataLeaseManager
@Override
@@ -166,18 +202,6 @@ public class DataNodeTableCache implements ITableCache {
}
}
- /**
- * The case that pre update Table and pre delete Table procedures targeting the same table are
- * executed serially by CN.
- *
- * <p>Consider the scenario:
- *
- * <ol>
- * <li>Drop the table first: DN executed pre update but missed the commit phase
- * <li>Create table second: DN executed pre update, which overwrites the result of the drop
- * table procedure in {@link #specialStatusMap}
- * </ol>
- */
@Override
public void preUpdateTable(String database, final TsTable table, final String oldName) {
database = PathUtils.unQualifyDatabaseName(database);
@@ -203,12 +227,12 @@ public class DataNodeTableCache implements ITableCache {
if (!table.getCachedNeedLastCache()) {
TableDeviceSchemaCache.getInstance().invalidateLastCache(database, table.getTableName());
}
-
if (table instanceof PreDeleteTsTable) {
if (databaseTableMap.containsKey(database)) {
databaseTableMap.get(database).remove(table.getTableName());
}
}
+
// If rename table
if (Objects.nonNull(oldName)) {
final TsTable oldTable = databaseTableMap.get(database).remove(oldName);
@@ -233,11 +257,17 @@ public class DataNodeTableCache implements ITableCache {
}
@Override
- public void rollbackUpdateTable(String database, final String tableName, final String oldName) {
+ public void rollbackUpdateTable(
+ String database,
+ final String tableName,
+ final String oldName,
+ TUpdateTree2TableView updateTree2TableViewInfo) {
database = PathUtils.unQualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS);
+ // for create or replace the view, only triggered when rollback
+ operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo);
// if rollback the drop table procedure, do nothing,
// wait for triggering the action of pull table from CN
final TsTable table = getTableFromSpecialStatusMap(database, tableName);
@@ -316,11 +346,16 @@ public class DataNodeTableCache implements ITableCache {
@Override
public void commitUpdateTable(
- String database, final String tableName, final @Nullable String oldName) {
+ String database,
+ final String tableName,
+ final @Nullable String oldName,
+ TUpdateTree2TableView updateTree2TableViewInfo) {
database = PathUtils.unQualifyDatabaseName(database);
readWriteLock.writeLock().lock();
try {
failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS);
+ operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo);
+ // for update tree-to-table view
final TsTable newTable = getTableFromSpecialStatusMap(database, tableName);
if (Objects.isNull(newTable)) {
LOGGER.info(
@@ -370,6 +405,116 @@ public class DataNodeTableCache implements ITableCache {
}
}
+ /**
+ * Applies the effect of a committed change on the tree-to-table view index. Every attribute the
+ * operation needs comes with the request: the table this DataNode caches may be a placeholder, so
+ * the entry cannot be computed from it.
+ */
+ private void operateOnTree2TableViewCache(
+ String database,
+ String tableName,
+ @Nullable String oldTableName,
+ TUpdateTree2TableView updateTree2TableViewInfo) {
+ if (Objects.isNull(updateTree2TableViewInfo)) {
+ return;
+ }
+ final TreeToTableViewTrieOperationType type =
+ TreeToTableViewTrieOperationType.getType(
+ updateTree2TableViewInfo.getTree2TableOperationType());
+ final boolean changed;
+ try {
+ switch (type) {
+ case REMOVE:
+ // Unregistering a view needs nothing but its name.
+ changed = treeToTableViewCache.removeView(database, tableName);
+ break;
+ case ALTER_RENAME:
+ // A rename keeps the place of the view.
+ changed = treeToTableViewCache.alterViewName(database, tableName, oldTableName);
+ break;
+ case ALTER_TAG:
+ changed =
+ treeToTableViewCache.alterViewProperties(
+ database,
+ tableName,
+ Collections.singletonMap(
+ TreeToTableViewReference.NUM_OF_TAGS,
+ String.valueOf(updateTree2TableViewInfo.getTagNum())));
+ break;
+ case ADD:
+ case ALTER_AUTO_SYNC:
+ changed =
+ treeToTableViewCache.addView(
+ database,
+ tableName,
+ updateTree2TableViewInfo.isRestrict,
+ updateTree2TableViewInfo.getTagNum(),
+ new PartialPath(updateTree2TableViewInfo.getTreePath()));
+ break;
+ default:
+ return;
+ }
+ } catch (final IllegalPathException e) {
+ LOGGER.warn(
+ DataNodeSchemaMessages
+ .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE,
+ database,
+ tableName,
+ e);
+ return;
+ }
+ if (changed) {
+ logTreeToTableViewChange(type, database, tableName, oldTableName, updateTree2TableViewInfo);
+ }
+ }
+
+ /** Logs the change an operation really applied to the tree-to-table view index. */
+ private static void logTreeToTableViewChange(
+ final TreeToTableViewTrieOperationType type,
+ final String database,
+ final String tableName,
+ final @Nullable String oldTableName,
+ final TUpdateTree2TableView updateTree2TableViewInfo) {
+ switch (type) {
+ case REMOVE:
+ TreeToTableViewCache.LOGGER.info(
+ DataNodeSchemaMessages.LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A,
+ database,
+ tableName);
+ break;
+ case ALTER_RENAME:
+ TreeToTableViewCache.LOGGER.info(
+ DataNodeSchemaMessages.LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41,
+ database,
+ tableName,
+ oldTableName);
+ break;
+ case ALTER_TAG:
+ TreeToTableViewCache.LOGGER.info(
+ DataNodeSchemaMessages
+ .LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984,
+ database,
+ tableName,
+ Collections.singletonMap(
+ TreeToTableViewReference.NUM_OF_TAGS,
+ String.valueOf(updateTree2TableViewInfo.getTagNum())));
+ break;
+ case ADD:
+ case ALTER_AUTO_SYNC:
+ TreeToTableViewCache.LOGGER.info(
+ DataNodeSchemaMessages
+ .LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2,
+ database,
+ tableName,
+ updateTree2TableViewInfo.getTreePath(),
+ updateTree2TableViewInfo.isRestrict,
+ updateTree2TableViewInfo.getTagNum());
+ break;
+ default:
+ break;
+ }
+ }
+
/**
* Repairs the cache after this DataNode missed the pre-update of a rename procedure. The
* ConfigNode has already renamed the table, so the new name can only be re-fetched, and the old
@@ -410,6 +555,7 @@ public class DataNodeTableCache implements ITableCache {
try {
databaseTableMap.remove(database);
specialStatusMap.remove(database);
+ treeToTableViewCache.removeViews(database);
instanceVersion.incrementAndGet();
} finally {
readWriteLock.writeLock().unlock();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java
index b87b6db7147..307df70b427 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.schemaengine.table;
import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy;
import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -36,10 +37,16 @@ public interface ITableCache {
void preUpdateTable(final String database, final TsTable table, final @Nullable String oldName);
void rollbackUpdateTable(
- final String database, final String tableName, final @Nullable String oldName);
+ final String database,
+ final String tableName,
+ final @Nullable String oldName,
+ TUpdateTree2TableView updateTree2TableViewInfo);
void commitUpdateTable(
- final String database, final String tableName, final @Nullable String oldName);
+ final String database,
+ final String tableName,
+ final @Nullable String oldName,
+ TUpdateTree2TableView updateTree2TableViewInfo);
/**
* @param database shouldn't start with `root.`
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java
new file mode 100644
index 00000000000..b1b675dfc71
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.schemaengine.table;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public interface ITreeToTableViewCache {
+
+ void init(Map<String, List<TsTable>> initialTables);
+
+ void clear();
+
+ /**
+ * Registers the view at {@code path}, the fixed prefix of its tree path pattern. {@code
+ * isRestrict} and {@code numOfTags} are the two attributes a device path is matched with, see
+ * {@link TreeToTableViewReference}.
+ */
+ boolean addView(
+ String database, String tableName, boolean isRestrict, int numOfTags, PartialPath path)
+ throws IllegalPathException;
+
+ boolean removeView(String database, String tableName);
+
+ /**
+ * Removes every view of the database, eg. when the database is dropped. The views are taken from
+ * the index itself, so the ones the table cache does not hold any more are removed as well.
+ */
+ void removeViews(String database);
+
+ boolean alterViewName(String database, String newName, String oldName);
+
+ /**
+ * Replaces the fields named in {@code properties} on the registered view. Only the restrict flag
+ * and the tag count can be replaced today, and a field whose key is absent keeps its value.
+ */
+ boolean alterViewProperties(String database, String tableName, Map<String, String> properties);
+
+ @Nullable
+ PartialPath getViewPath(String database, String tableName);
+
+ Set<TreeToTableViewReference> getTreeToTableViews(PartialPath path);
+
+ boolean hasView();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java
new file mode 100644
index 00000000000..473b2266f54
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.schemaengine.table;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.exception.MetadataLeaseFencedException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternUtil;
+import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory;
+import org.apache.iotdb.commons.schema.table.TreeViewSchema;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.db.i18n.DataNodeSchemaMessages;
+import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager;
+import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode;
+import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader;
+import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference;
+
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT;
+
+/**
+ * The index of the tree to table views on DataNode, keyed by the fixed prefix of each view's tree
+ * path pattern.
+ *
+ * <p>Only DataNodeTableCache writes this index, and it does so while holding its own write lock, so
+ * every write here also holds the write lock of this cache and the lock order is always
+ * DataNodeTableCache's write lock and then this cache's write lock. A reader only takes this
+ * cache's read lock, so the two locks can never form a cycle.
+ */
+public class TreeToTableViewCache implements ITreeToTableViewCache {
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(TreeToTableViewCache.class);
+
+ private final IMNodeFactory<IMemMNode> nodeFactory =
+ MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory();
+ // This index describes path prefixes, not the database/device roles in a SchemaRegion. clear()
+ // replaces it with a fresh one.
+ private IMemMNode rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT);
+
+ private final Map<IMemMNode, Set<TreeToTableViewReference>> viewReferences =
+ new IdentityHashMap<>();
+ private final Map<Pair<String, String>, PartialPath> viewPaths = new HashMap<>();
+ private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+
+ /**
+ * Whether any view is registered. A cluster without any view answers "no view covers this path"
+ * without taking the lock or walking the trie. It is written only under the write lock, and read
+ * without any lock.
+ */
+ private volatile boolean hasView = false;
+
+ private TreeToTableViewCache() {}
+
+ @Override
+ public void clear() {
+ readWriteLock.writeLock().lock();
+ try {
+ // A node is only reachable from its parent, so a fresh root drops the whole trie at once.
+ rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT);
+ viewReferences.clear();
+ viewPaths.clear();
+ hasView = false;
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void init(final Map<String, List<TsTable>> initialTables) {
+ readWriteLock.writeLock().lock();
+ try {
+ // The index is rebuilt from scratch, so the entry of a view which is gone does not survive.
+ clear();
+ initialTables.forEach(
+ (database, tables) -> tables.forEach(table -> registerView(database, table)));
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Registers the view at the path of its tree path pattern. The caller has already picked the
+ * tables which belong in the index, see DataNodeTableCache#collectInitialViews.
+ */
+ private void registerView(final String database, final TsTable table) {
+ final Optional<String> treePath = table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN);
+ if (treePath.isEmpty()) {
+ return;
+ }
+ try {
+ addView(
+ database,
+ table.getTableName(),
+ TreeViewSchema.isRestrict(table),
+ table.getTagNum(),
+ new PartialPath(treePath.get()));
+ } catch (final IllegalPathException e) {
+ LOGGER.warn(
+ DataNodeSchemaMessages
+ .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E,
+ database,
+ table.getTableName(),
+ e);
+ }
+ }
+
+ /**
+ * Register a view at a fixed prefix path.
+ *
+ * @parm Partial path, eg: root.db.dev01.**;
+ * @return true if a new reference was registered; false if it was already present
+ */
+ @Override
+ public boolean addView(
+ final String database,
+ final String tableName,
+ final boolean isRestrict,
+ final int numOfTags,
+ final PartialPath path)
+ throws IllegalPathException {
+ int prefixLength = checkPartialPath(path);
+ final String[] nodes = path.getNodes();
+ readWriteLock.writeLock().lock();
+ try {
+ final PartialPath registeredPath = viewPaths.get(new Pair<>(database, tableName));
+ if (Objects.nonNull(registeredPath) && !Objects.equals(registeredPath, path)) {
+ // A view is registered at exactly one path, so the entry left at the previous path has to
+ // go, or removing this view by name later would miss it.
+ removeView(database, tableName);
+ }
+ viewPaths.put(new Pair<>(database, tableName), path);
+
+ IMemMNode current = rootNode;
+ for (int i = 1; i < prefixLength; i++) {
+ IMemMNode child = current.getChild(nodes[i]);
+ if (child == null) {
+ child = current.addChild(nodes[i], nodeFactory.createInternalMNode(current, nodes[i]));
+ }
+ current = child;
+ }
+
+ final TreeToTableViewReference reference =
+ new TreeToTableViewReference(database, tableName, isRestrict, numOfTags);
+ final Set<TreeToTableViewReference> references =
+ viewReferences.computeIfAbsent(current, node -> new HashSet<>());
+ final boolean added = !references.remove(reference);
+ references.add(reference);
+ // whether the reference is new or not, this path is covered by a view now
+ hasView = true;
+ return added;
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Removes a view from the trie.
+ *
+ * @return true if the view was registered and has been removed; false otherwise
+ */
+ @Override
+ public boolean removeView(String database, String tableName) {
+ readWriteLock.writeLock().lock();
+ try {
+ return removeViewLocked(database, tableName);
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Removes every view of the database. A database of the same name may be created again with views
+ * which have nothing to do with the dropped ones, so the entries of the old database have to go
+ * together with it.
+ */
+ @Override
+ public void removeViews(final String database) {
+ readWriteLock.writeLock().lock();
+ try {
+ // viewPaths is the record of what is registered, so the views which the table cache does not
+ // hold any more are removed here as well.
+ final List<String> tableNames = new ArrayList<>();
+ for (final Pair<String, String> view : viewPaths.keySet()) {
+ if (Objects.equals(database, view.getLeft())) {
+ tableNames.add(view.getRight());
+ }
+ }
+ tableNames.forEach(tableName -> removeViewLocked(database, tableName));
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ /** The body of {@link #removeView}, which the caller runs while holding the write lock. */
+ private boolean removeViewLocked(String database, String tableName) {
+ final RegisteredView registered = getRegisteredView(database, tableName);
+ if (Objects.isNull(registered)) {
+ return false;
+ }
+
+ registered.references.remove(registered.reference);
+ if (registered.references.isEmpty()) {
+ viewReferences.remove(registered.node);
+ pruneNodesWithoutView(registered.node);
+ hasView = !viewReferences.isEmpty();
+ }
+ viewPaths.remove(new Pair<>(database, tableName));
+ return true;
+ }
+
+ @Override
+ public boolean alterViewName(String database, String newName, String oldName) {
+ readWriteLock.writeLock().lock();
+ try {
+ final RegisteredView registered = getRegisteredView(database, oldName);
+ if (Objects.isNull(registered)) {
+ LOGGER.warn(
+ DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902,
+ database,
+ oldName);
+ return false;
+ }
+
+ final TreeToTableViewReference oldReference = registered.reference;
+ registered.references.remove(oldReference);
+ registered.references.add(
+ new TreeToTableViewReference(
+ database, newName, oldReference.isRestrict(), oldReference.getNumOfTags()));
+ // A rename does not move the view, so the path of the old name follows the new name.
+ final PartialPath path = viewPaths.remove(new Pair<>(database, oldName));
+ viewPaths.put(new Pair<>(database, newName), path);
+ return true;
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public boolean alterViewProperties(
+ final String database, final String tableName, final Map<String, String> properties) {
+ readWriteLock.writeLock().lock();
+ try {
+ final RegisteredView registered = getRegisteredView(database, tableName);
+ if (Objects.isNull(registered)) {
+ LOGGER.warn(
+ DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902,
+ database,
+ tableName);
+ return false;
+ }
+
+ final TreeToTableViewReference oldReference = registered.reference;
+ final String restrict = properties.get(TreeViewSchema.RESTRICT);
+ final String numOfTags = properties.get(TreeToTableViewReference.NUM_OF_TAGS);
+ registered.references.remove(oldReference);
+ registered.references.add(
+ new TreeToTableViewReference(
+ database,
+ tableName,
+ Objects.isNull(restrict) ? oldReference.isRestrict() : Boolean.parseBoolean(restrict),
+ Objects.isNull(numOfTags)
+ ? oldReference.getNumOfTags()
+ : Integer.parseInt(numOfTags)));
+ return true;
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * The place a registered view lives: the trie node it sits at, the references of that node, and
+ * the reference of the view itself.
+ */
+ private static class RegisteredView {
+ IMemMNode node;
+ Set<TreeToTableViewReference> references;
+ TreeToTableViewReference reference;
+
+ private RegisteredView(
+ final IMemMNode node,
+ final Set<TreeToTableViewReference> references,
+ final TreeToTableViewReference reference) {
+ this.node = node;
+ this.references = references;
+ this.reference = reference;
+ }
+ }
+
+ /**
+ * The registered view of these names, or null when no view of them is registered. The node may
+ * also exist as an intermediate node of another view's path, without any view of its own.
+ */
+ private RegisteredView getRegisteredView(final String database, final String tableName) {
+ final IMemMNode node = getViewNode(database, tableName);
+ final Set<TreeToTableViewReference> references =
+ Objects.isNull(node) ? null : viewReferences.get(node);
+ if (Objects.isNull(references)) {
+ return null;
+ }
+ final TreeToTableViewReference reference = findReference(references, database, tableName);
+ return Objects.isNull(reference) ? null : new RegisteredView(node, references, reference);
+ }
+
+ /** The trie node the view of these names is registered at, or null if it is not registered. */
+ private IMemMNode getViewNode(final String database, final String tableName) {
+ final PartialPath path = viewPaths.get(new Pair<>(database, tableName));
+ if (Objects.isNull(path)) {
+ return null;
+ }
+ final String[] nodes = path.getNodes();
+ return findNode(nodes, nodes.length - 1);
+ }
+
+ /** The reference of the view of these names in {@code references}, or null if it holds none. */
+ private static TreeToTableViewReference findReference(
+ final Set<TreeToTableViewReference> references,
+ final String database,
+ final String tableName) {
+ for (final TreeToTableViewReference reference : references) {
+ if (Objects.equals(database, reference.getDatabaseName())
+ && Objects.equals(tableName, reference.getTableName())) {
+ return reference;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @param path the device path, eg, root.db01.plant01.dev01
+ * @return the views covering the device path, empty if there is none
+ */
+ @Override
+ public Set<TreeToTableViewReference> getTreeToTableViews(PartialPath path) {
+ // A fenced DataNode must not answer from its possibly stale index, not even with "no view
+ // covers this path": the caller has to know that the cache is unavailable rather than empty.
+ failIfMetadataLeaseFenced();
+ if (!hasView) {
+ // the answer for every path when no view is registered at all
+ return Collections.emptySet();
+ }
+ final String[] nodes = path.getNodes();
+ readWriteLock.readLock().lock();
+ try {
+ Set<TreeToTableViewReference> result = new HashSet<>();
+ IMemMNode current = rootNode;
+ int currentLevel = 1;
+ while (current != null && currentLevel < (nodes.length + 1)) {
+ // maybe has the qualified nodes, check and collect
+ if (viewReferences.containsKey(current)) {
+ checkAndCollectView(viewReferences.get(current), nodes.length - currentLevel, result);
+ }
+ if (currentLevel == nodes.length) {
+ break;
+ }
+ current = current.getChild(nodes[currentLevel]);
+ currentLevel++;
+ }
+ return result;
+ } finally {
+ readWriteLock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public boolean hasView() {
+ return hasView;
+ }
+
+ /**
+ * The path a view is registered at, so that a caller which knows only the names of the view can
+ * still locate and remove it.
+ *
+ * @return the path the view is registered at, or null if no view of these names is registered
+ */
+ @Override
+ public @Nullable PartialPath getViewPath(final String database, final String tableName) {
+ readWriteLock.readLock().lock();
+ try {
+ return viewPaths.get(new Pair<>(database, tableName));
+ } finally {
+ readWriteLock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Walks the fixed prefix of a checked path, which is the path without the leading "root" and the
+ * trailing multi level wildcard, eg, nodes of root.db.dev01.** give the node of root.db.dev01.
+ *
+ * @return the node the prefix ends at, or null if the trie does not hold that prefix
+ */
+ private IMemMNode findNode(final String[] nodes, final int prefixLength) {
+ IMemMNode current = rootNode;
+ for (int i = 1; i < prefixLength && current != null; i++) {
+ current = current.getChild(nodes[i]);
+ }
+ return current;
+ }
+
+ /**
+ * Drops the nodes which have neither a child nor a view of their own any more, starting at {@code
+ * node} and walking up to the first node that is still needed. Removing a view is rare, so the
+ * walk is affordable.
+ */
+ private void pruneNodesWithoutView(final IMemMNode node) {
+ IMemMNode current = node;
+ while (current != rootNode) {
+ final IMemMNode parent = current.getParent();
+ if (parent == null
+ || !current.getChildren().isEmpty()
+ || viewReferences.containsKey(current)) {
+ return;
+ }
+ parent.deleteChild(current.getName());
+ current = parent;
+ }
+ }
+
+ void failIfMetadataLeaseFenced() {
+ MetadataLeaseManager.getInstance()
+ .failIfMetadataLeaseFenced(
+ MetadataLeaseFencedException.LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS);
+ }
+
+ private static final class TreeToTableViewCacheHolder {
+ private static final TreeToTableViewCache INSTANCE = new TreeToTableViewCache();
+
+ private TreeToTableViewCacheHolder() {}
+ }
+
+ public static ITreeToTableViewCache getInstance() {
+ return TreeToTableViewCacheHolder.INSTANCE;
+ }
+
+ // the path has to be like root.db.dev01.**;
+ private int checkPartialPath(PartialPath path) throws IllegalPathException {
+ final String[] nodes = path.getNodes();
+ if (nodes.length == 0 || !PATH_ROOT.equals(nodes[0])) {
+ throw new IllegalPathException(path.getFullPath());
+ }
+ if (!MULTI_LEVEL_PATH_WILDCARD.equals(nodes[nodes.length - 1])) {
+ throw new IllegalPathException(path.getFullPath());
+ }
+ final int prefixLength = nodes.length - 1;
+ // Validate before modifying the trie. Wildcards inside the fixed prefix are not supported.
+ for (int i = 1; i < prefixLength; i++) {
+ if (nodes[i] == null || nodes[i].isEmpty() || PathPatternUtil.hasWildcard(nodes[i])) {
+ throw new IllegalPathException(path.getFullPath());
+ }
+ }
+ return prefixLength;
+ }
+
+ private void checkAndCollectView(
+ Set<TreeToTableViewReference> treeToTableViewReferences,
+ int remainingLevel,
+ Set<TreeToTableViewReference> result) {
+ treeToTableViewReferences.forEach(
+ view -> {
+ int numOfTags = view.getNumOfTags();
+ boolean match =
+ view.isRestrict() ? numOfTags == remainingLevel : numOfTags >= remainingLevel;
+ if (match) {
+ result.add(view);
+ }
+ });
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java
new file mode 100644
index 00000000000..9a937fcd315
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.schemaengine.table;
+
+import org.apache.iotdb.commons.client.AbstractConfigNodeInfo;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.consensus.ConfigRegionId;
+import org.apache.iotdb.commons.exception.MetadataLeaseFencedException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType;
+import org.apache.iotdb.commons.schema.table.TreeViewSchema;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp;
+import org.apache.iotdb.db.i18n.DataNodeSchemaMessages;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator;
+import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HexFormat;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Adds to the tree-to-table views the columns which the measurements a schema write has created
+ * need.
+ *
+ * <p>It is the automatic counterpart of the {@code ALTER VIEW ... ADD COLUMN} statement: the
+ * columns come from the measurements the schema write has just created, instead of from the
+ * statement itself.
+ */
+public class ViewColumnAutoSyncTrigger {
+
+ // Public so that the schema write path, which invokes sync and catches its failures, logs them
+ // with the logger of the feature; logback routes this logger to its own error file.
+ public static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSyncTrigger.class);
+ private static final IClientManager<ConfigRegionId, ConfigNodeClient> CONFIG_NODE_CLIENT_MANAGER =
+ ConfigNodeClientManager.getInstance();
+
+ public static final String QUERY_ID_PREFIX = "tree_to_table_add_column";
+ public static final String EXTEND_DEVICE_TEMPLATE_ID_PREFIX = "template_";
+
+ /** How many devices, and how many measurements per device, a fenced lease log line shows. */
+ private static final int MAX_LOGGED_DEVICES = 10;
+
+ private static final int MAX_LOGGED_MEASUREMENTS_PER_DEVICE = 10;
+
+ /**
+ * Adds the columns the views covering the given devices need.
+ *
+ * <p>It is a best-effort side effect of the write which triggers it: every failure is logged here
+ * and none of them is propagated, so that the write operation is never affected.
+ *
+ * @param device2Measurements the measurements the schema write really created, eg. device01 ->
+ * {s1, s2, s3}; device02 -> {s3, s4}. Only they get a column, so that a creation which failed
+ * cannot leave a column behind.
+ * @param operationId the planNodeId of the plan which triggers this operation, for the log
+ * messages only: it is a per-query index, so it does not identify the operation
+ */
+ public static void sync(
+ final Map<PartialPath, List<Pair<String, TSDataType>>> device2Measurements,
+ final String operationId) {
+ if (device2Measurements == null || device2Measurements.isEmpty()) {
+ return;
+ }
+
+ // the views to be added columns
+ final Map<TreeToTableViewReference, Set<Pair<String, TSDataType>>> view2Measurements;
+ try {
+ view2Measurements = collectViewMeasurements(device2Measurements);
+ } catch (final MetadataLeaseFencedException e) {
+ // The DataNode is isolated: neither the index nor the table cache it consults may be used.
+ // Give up this round and record the measurements which cannot be added while it is fenced.
+ LOGGER.error(
+ DataNodeSchemaMessages
+ .LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44,
+ formatDeviceMeasurements(device2Measurements));
+ return;
+ }
+ if (view2Measurements.isEmpty()) {
+ return;
+ }
+
+ try (ConfigNodeClient configNodeClient =
+ CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) {
+ List<TAlterOrDropTableReq> updateViewReqs =
+ view2Measurements.entrySet().stream()
+ .map(ViewColumnAutoSyncTrigger::toAlterTableReq)
+ .toList();
+
+ TAutoSyncViewColumnsResp resp =
+ configNodeClient.autoSyncViewColumns(new TAutoSyncViewColumnsReq(updateViewReqs));
+ if (resp == null || resp.getStatus() == null) {
+ LOGGER.error(
+ DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26, operationId);
+ return;
+ }
+ if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ LOGGER.error(
+ DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC,
+ operationId,
+ resp.getStatus());
+ }
+ // the views are updated one by one in ConfigNode, so the failed ones are reported one by one
+ if (resp.getFailedViews() != null) {
+ for (TAutoSyncViewColumnsFailure failedView : resp.getFailedViews()) {
+ LOGGER.error(
+ DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144,
+ failedView.getDatabase(),
+ failedView.getTableName(),
+ operationId,
+ failedView.getStatus());
+ }
+ }
+ } catch (ClientManagerException | TException e) {
+ LOGGER.warn(
+ DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652, operationId, e);
+ }
+ }
+
+ /**
+ * A bounded, human readable form of the measurements, so that the log line of a batch write stays
+ * short: at most {@link #MAX_LOGGED_DEVICES} devices and {@link
+ * #MAX_LOGGED_MEASUREMENTS_PER_DEVICE} measurements per device are written, the rest is
+ * summarized.
+ */
+ private static String formatDeviceMeasurements(
+ final Map<PartialPath, List<Pair<String, TSDataType>>> device2Measurements) {
+ final StringBuilder builder = new StringBuilder("{");
+ int loggedDevices = 0;
+ for (final Map.Entry<PartialPath, List<Pair<String, TSDataType>>> entry :
+ device2Measurements.entrySet()) {
+ if (loggedDevices == MAX_LOGGED_DEVICES) {
+ builder.append("... (").append(device2Measurements.size()).append(" devices in total)}");
+ return builder.toString();
+ }
+ if (loggedDevices > 0) {
+ builder.append(", ");
+ }
+ builder.append(entry.getKey().getFullPath()).append("=[");
+ final List<Pair<String, TSDataType>> measurements = entry.getValue();
+ for (int i = 0; i < measurements.size(); i++) {
+ if (i == MAX_LOGGED_MEASUREMENTS_PER_DEVICE) {
+ builder.append("... (").append(measurements.size()).append(" measurements in total)");
+ break;
+ }
+ if (i > 0) {
+ builder.append(", ");
+ }
+ builder
+ .append(measurements.get(i).getLeft())
+ .append(':')
+ .append(measurements.get(i).getRight());
+ }
+ builder.append(']');
+ loggedDevices++;
+ }
+ return builder.append('}').toString();
+ }
+
+ /**
+ * @return the measurements each view covering the given devices still needs; eg: view01 -> {s1,
+ * s2, s3}, view02 -> {s1, s2, s3, s4}
+ */
+ private static Map<TreeToTableViewReference, Set<Pair<String, TSDataType>>>
+ collectViewMeasurements(
+ Map<PartialPath, List<Pair<String, TSDataType>>> device2Measurements) {
+
+ // deduplicate the measurement by set, the measurement with same name and dataType
+ // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32)
+ // the measurements which already have a column in the view are dropped below
+ final Map<TreeToTableViewReference, Set<Pair<String, TSDataType>>> view2Measurements =
+ new HashMap<>();
+
+ for (Map.Entry<PartialPath, List<Pair<String, TSDataType>>> entry :
+ device2Measurements.entrySet()) {
+ PartialPath path = entry.getKey();
+ Set<TreeToTableViewReference> treeToTableViews =
+ TreeToTableViewCache.getInstance().getTreeToTableViews(path);
+
+ treeToTableViews.forEach(
+ view ->
+ view2Measurements
+ .computeIfAbsent(view, key -> new LinkedHashSet<>())
+ .addAll(entry.getValue()));
+ }
+
+ // A measurement the view already has a column for is not needed again: ConfigNode would drop
+ // it in its column check and reject the request when nothing is left, which costs an RPC and a
+ // persisted procedure for nothing. A view the table cache does not hold is left as it is, as
+ // the DataNode cannot tell which columns are missing then.
+ final Iterator<Map.Entry<TreeToTableViewReference, Set<Pair<String, TSDataType>>>> iterator =
+ view2Measurements.entrySet().iterator();
+ while (iterator.hasNext()) {
+ final Map.Entry<TreeToTableViewReference, Set<Pair<String, TSDataType>>> viewEntry =
+ iterator.next();
+ final TsTable view = getCachedView(viewEntry.getKey());
+ if (Objects.isNull(view)) {
+ continue;
+ }
+ removeExistingMeasurements(view, viewEntry.getValue());
+ if (viewEntry.getValue().isEmpty()) {
+ iterator.remove();
+ }
+ }
+ return view2Measurements;
+ }
+
+ /** The view as the DataNode caches it, or null when the cache does not hold it. */
+ private static TsTable getCachedView(final TreeToTableViewReference view) {
+ return DataNodeTableCache.getInstance()
+ .getTable(view.getDatabaseName(), view.getTableName(), false);
+ }
+
+ /** Drops the measurements which the view already has a column for. */
+ private static void removeExistingMeasurements(
+ final TsTable view, final Set<Pair<String, TSDataType>> measurements) {
+ // The same test ConfigNode applies before it adds the columns: the column name, or the source
+ // measurement a FROM column references. A measurement the cache does not know yet is kept, as
+ // the cache may lag behind ConfigNode.
+ final Set<String> sourceNames = TreeViewSchema.getSourceNames(view);
+ measurements.removeIf(
+ measurement ->
+ Objects.nonNull(view.getColumnSchema(measurement.getLeft().toLowerCase(Locale.ROOT)))
+ || sourceNames.contains(measurement.getLeft()));
+ }
+
+ private static TAlterOrDropTableReq toAlterTableReq(
+ Map.Entry<TreeToTableViewReference, Set<Pair<String, TSDataType>>> view2Measurements) {
+ final TreeToTableViewReference view = view2Measurements.getKey();
+ final List<TsTableColumnSchema> columnSchemas =
+ view2Measurements.getValue().stream()
+ .map(ViewColumnAutoSyncTrigger::toColumnSchema)
+ .toList();
+ TAlterOrDropTableReq tAlterOrDropTableReq =
+ new TAlterOrDropTableReq(
+ view.getDatabaseName(),
+ view.getTableName(),
+ buildQueryId(
+ columnFingerprint(columnSchemas), view.getDatabaseName(), view.getTableName()),
+ AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue(),
+ ByteBuffer.wrap(TsTableColumnSchemaUtil.serialize(columnSchemas)));
+ tAlterOrDropTableReq.setIsView(true);
+ return tAlterOrDropTableReq;
+ }
+
+ // transform the upperCase to the from-attribute
+ private static TsTableColumnSchema toColumnSchema(final Pair<String, TSDataType> measurement) {
+ final String measurementName = measurement.getLeft();
+ final String from =
+ measurementName.chars().anyMatch(Character::isUpperCase) ? measurementName : null;
+ return TableHeaderSchemaValidator.generateColumnSchema(
+ TsTableColumnCategory.FIELD,
+ measurementName.toLowerCase(Locale.ROOT),
+ measurement.getRight(),
+ null,
+ from);
+ }
+
+ // leader and follower would build the same queryId when apply the same log; so CN could recognize
+ // the same add view column request. The database and table names may contain any character, so
+ // their lengths are written out to keep the parts apart: without them, the views a_b.c and
+ // a.b_c would share an id and the request for the second one would be taken for a repeat of the
+ // first one's.
+ private static String buildQueryId(String columnFingerprint, String database, String table) {
+ return String.join(
+ "_",
+ QUERY_ID_PREFIX,
+ columnFingerprint,
+ String.valueOf(database.length()),
+ database,
+ String.valueOf(table.length()),
+ table);
+ }
+
+ /**
+ * The columns are sorted first so that the same column set always yields the same fingerprint, no
+ * matter in which order the measurements were collected.
+ */
+ private static String columnFingerprint(final List<TsTableColumnSchema> columnSchemas) {
+ final List<TsTableColumnSchema> sortedColumnSchemas = new ArrayList<>(columnSchemas);
+ sortedColumnSchemas.sort(Comparator.comparing(TsTableColumnSchema::getColumnName));
+ return digestOf(TsTableColumnSchemaUtil.serialize(sortedColumnSchemas));
+ }
+
+ private static String digestOf(final byte[] content) {
+ try {
+ // 4 bytes are enough here, the number of distinct column sets of one view is rather small
+ return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content), 0, 4);
+ } catch (final NoSuchAlgorithmException e) {
+ // SHA-256 is supported by every Java platform
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java
index bf2f5c21e6a..8b3217bad2f 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java
@@ -595,7 +595,7 @@ public class IoTConsensusShutdownTest {
DataNodeTableCache.getInstance()
.preUpdateTable(DATABASE, StatementTestUtils.genTsTable(), null);
DataNodeTableCache.getInstance()
- .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null);
+ .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null, null);
assertEquals(
TSStatusCode.SUCCESS_STATUS.getStatusCode(),
stateMachine
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java
index d60a8f2cc3c..98ef2dc61c0 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java
@@ -290,7 +290,7 @@ public class SchemaRegionSimpleRecoverTest extends AbstractSchemaRegionTest {
new TagColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType())));
testTable.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING));
DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null);
schemaRegion.updateTableDeviceAttribute(
new TableDeviceAttributeUpdateNode(
@@ -337,7 +337,7 @@ public class SchemaRegionSimpleRecoverTest extends AbstractSchemaRegionTest {
testTable.addColumnSchema(new AttributeColumnSchema("attr", TSDataType.STRING));
testTable.addColumnSchema(new FieldColumnSchema("field", TSDataType.STRING));
DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null);
try {
final ISchemaRegion schemaRegion = getSchemaRegion(database, 0);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
index 744bda9ce5b..cc7f9798471 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
@@ -159,7 +159,7 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
TreeViewSchema.TREE_PATH_PATTERN,
NON_ALIGNED_TREE_DEVICE_VIEW_SCAN_OPERATOR_TREE_TEST + ".**");
DataNodeTableCache.getInstance().preUpdateTable(tableDbName, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null, null);
}
@After
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
index 9c9918ddbc8..bc4fd02b192 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
@@ -1149,7 +1149,7 @@ public class AnalyzerTest {
tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING));
tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64));
DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, table, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null);
try {
final Update statement =
@@ -1211,7 +1211,7 @@ public class AnalyzerTest {
tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING));
tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64));
DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, table, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null);
try {
assertUpdateSemanticException(
@@ -1276,7 +1276,7 @@ public class AnalyzerTest {
tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING));
tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING));
DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, table, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null);
try {
final ShowDevice statement = new ShowDevice(database, table);
@@ -1316,7 +1316,7 @@ public class AnalyzerTest {
tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING));
tsTable.addColumnSchema(new AttributeColumnSchema("attr2", TSDataType.STRING));
DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, table, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null);
try {
final SqlParser sqlParser = new SqlParser();
@@ -1370,7 +1370,7 @@ public class AnalyzerTest {
private Metadata mockMetadataForInsertion() {
final TsTable tsTable = StatementTestUtils.genTsTable();
DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, table, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null);
return new TestMetadata() {
@Override
public Optional<TableSchema> validateTableHeaderSchema4TsFile(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java
index 0c154850ca5..74667017f25 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java
@@ -73,7 +73,8 @@ public class TreeViewTest {
tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + "");
tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test" + ".**");
DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null);
+ DataNodeTableCache.getInstance()
+ .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null);
}
@After
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
index 2b8909f3861..a59d3683902 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
@@ -107,10 +107,10 @@ public class TableDeviceSchemaCacheTest {
new FieldColumnSchema(
measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP));
DataNodeTableCache.getInstance().preUpdateTable(database1, testTable1, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null);
DataNodeTableCache.getInstance().preUpdateTable(database2, testTable1, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null, null);
final TsTable testTable2 = new TsTable(table2);
columnHeaderList.forEach(
@@ -139,7 +139,7 @@ public class TableDeviceSchemaCacheTest {
new FieldColumnSchema(
measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP));
DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null, null);
final TsTable testTableNeedLastCacheDisabled = new TsTable(tableNeedLastCacheDisabled);
columnHeaderList.forEach(
@@ -173,7 +173,8 @@ public class TableDeviceSchemaCacheTest {
TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString());
DataNodeTableCache.getInstance()
.preUpdateTable(database1, testTableNeedLastCacheDisabled, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database1, tableNeedLastCacheDisabled, null);
+ DataNodeTableCache.getInstance()
+ .commitUpdateTable(database1, tableNeedLastCacheDisabled, null, null);
}
@AfterClass
@@ -628,7 +629,7 @@ public class TableDeviceSchemaCacheTest {
new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1));
disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString());
DataNodeTableCache.getInstance().preUpdateTable(database1, disabledTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null);
try {
Assert.assertNull(cache.getLastEntry(database1, deviceID, measurement1));
@@ -636,7 +637,7 @@ public class TableDeviceSchemaCacheTest {
final TsTable enabledTable = new TsTable(disabledTable);
enabledTable.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY);
DataNodeTableCache.getInstance().preUpdateTable(database1, enabledTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null);
}
}
@@ -654,7 +655,7 @@ public class TableDeviceSchemaCacheTest {
new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1));
enabledTable.renameTable(tableName);
DataNodeTableCache.getInstance().preUpdateTable(database, enabledTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null);
final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance();
updateLastCache4Query(cache, database, deviceID, measurements, values);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java
index 91b16f17d5e..305f6dd706c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java
@@ -48,7 +48,8 @@ public class TopKRuntimeFilterTreeViewPlanTest {
tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + "");
tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test.**");
DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null);
+ DataNodeTableCache.getInstance()
+ .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null);
}
@After
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java
index bf42e644609..e0ddc58039f 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java
@@ -99,7 +99,7 @@ public class InsertStatementTest {
TsTable tsTable = convertTableSchemaToTsTable(tableSchema);
DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null);
+ DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null);
when(metadata.validateTableHeaderSchema4TsFile(
any(String.class),
@@ -252,7 +252,7 @@ public class InsertStatementTest {
TsTable tsTable = convertTableSchemaToTsTable(tableSchema);
DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null);
+ DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null);
when(metadata.validateTableHeaderSchema4TsFile(
any(String.class),
@@ -288,7 +288,7 @@ public class InsertStatementTest {
TsTable tsTable = convertTableSchemaToTsTable(tableSchema);
DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null);
+ DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null);
when(metadata.validateTableHeaderSchema4TsFile(
any(String.class),
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java
index f9e92207f34..a14d7b21244 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java
@@ -115,7 +115,7 @@ public class SchemaMemMetricTableTest {
table.addColumnSchema(
new FieldColumnSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP));
tableCache.preUpdateTable(DATABASE, table, null);
- tableCache.commitUpdateTable(DATABASE, TABLE, null);
+ tableCache.commitUpdateTable(DATABASE, TABLE, null, null);
final MemSchemaRegionStatistics statistics =
new MemSchemaRegionStatistics(SCHEMA_REGION_ID, new MemSchemaEngineStatistics());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java
index 865ef633248..91cfff1d635 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java
@@ -81,8 +81,8 @@ public class DataNodeTableCacheLeaseTest {
nowNanos.addAndGet((T_FENCE_MS + 1) * 1_000_000L);
assertLeaseFenced(() -> tableCache.preUpdateTable("root.db", table, null));
- assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null));
- assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null));
+ assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null, null));
+ assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null, null));
}
@Test
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java
index db9b36c7555..18ee416d2e8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java
@@ -60,12 +60,12 @@ public class DataNodeTableCacheTest {
final TsTable initial = createTable(TABLE_NAME);
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, initial, null);
Assert.assertNull(cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME));
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME));
final TsTable updated = new TsTable(initial);
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, updated, null);
Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME));
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
Assert.assertSame(updated, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME));
} finally {
cache.invalid(TABLE_CACHE_TEST_DATABASE);
@@ -100,8 +100,8 @@ public class DataNodeTableCacheTest {
try {
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null);
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
Assert.assertEquals(
TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName());
@@ -117,8 +117,8 @@ public class DataNodeTableCacheTest {
try {
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null);
- cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
+ cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
Assert.assertNull(cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, false));
} finally {
@@ -133,10 +133,10 @@ public class DataNodeTableCacheTest {
cache.invalid(TABLE_CACHE_TEST_DATABASE);
try {
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null);
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null);
cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(newTableName), TABLE_NAME);
- cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME);
+ cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME, null);
Assert.assertEquals(
TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName());
@@ -164,9 +164,9 @@ public class DataNodeTableCacheTest {
TsTableInternalRPCUtil.serializeTableInitializationInfo(usingTables, specialTables));
// The commit is the only sign of the rename, and the HA broadcast may send it more than once.
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null);
assertRenameRepaired(cache, oldName, newName);
- cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName);
+ cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null);
assertRenameRepaired(cache, oldName, newName);
} finally {
cache.invalid(TABLE_CACHE_TEST_DATABASE);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java
new file mode 100644
index 00000000000..58ec3c7b4aa
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.schemaengine.table;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode;
+import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Set;
+
+public class TreeToTableViewCacheTest {
+
+ /**
+ * The cache is a singleton, so every case registers its own views and removes them again, and
+ * every case uses its own database.
+ */
+ private final ITreeToTableViewCache viewCache = TreeToTableViewCache.getInstance();
+
+ @Test
+ public void testAddAlterAndRemoveView() throws Exception {
+ final String database = "add_test";
+ final PartialPath viewPath = new PartialPath("root.add_test.dev01.**");
+ final PartialPath devicePath = new PartialPath("root.add_test.dev01");
+ try {
+ Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty());
+
+ Assert.assertTrue(viewCache.addView(database, "view1", false, 0, viewPath));
+ // the same view at the same path twice does not add a second reference
+ Assert.assertFalse(viewCache.addView(database, "view1", false, 0, viewPath));
+ Assert.assertTrue(isCovered(devicePath, "view1"));
+
+ Assert.assertTrue(viewCache.alterViewName(database, "view2", "view1"));
+ Assert.assertFalse(isCovered(devicePath, "view1"));
+ Assert.assertTrue(isCovered(devicePath, "view2"));
+
+ Assert.assertTrue(viewCache.removeView(database, "view2"));
+ Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty());
+ // the view is gone, so removing it again fails
+ Assert.assertFalse(viewCache.removeView(database, "view2"));
+ } finally {
+ viewCache.removeView(database, "view1");
+ viewCache.removeView(database, "view2");
+ }
+ }
+
+ @Test
+ public void testRemovingOneViewKeepsTheOthers() throws Exception {
+ final String database = "remove_test";
+ final PartialPath viewPath = new PartialPath("root.remove_test.dev01.**");
+ final PartialPath devicePath = new PartialPath("root.remove_test.dev01");
+ try {
+ Assert.assertTrue(viewCache.addView(database, "view3", false, 0, viewPath));
+ Assert.assertTrue(viewCache.addView(database, "view4", false, 0, viewPath));
+
+ Assert.assertTrue(viewCache.removeView(database, "view3"));
+
+ final Set<TreeToTableViewReference> views = viewCache.getTreeToTableViews(devicePath);
+ Assert.assertEquals(1, views.size());
+ Assert.assertEquals("view4", views.iterator().next().getTableName());
+ } finally {
+ viewCache.removeView(database, "view3");
+ viewCache.removeView(database, "view4");
+ }
+ }
+
+ /** The nodes created for the path of the removed view have to be dropped together with it. */
+ @Test
+ public void testRemoveViewPrunesTheTrie() throws Exception {
+ final String database = "prune_test";
+ final PartialPath viewPath = new PartialPath("root.prune_test.dev01.**");
+ final PartialPath devicePath = new PartialPath("root.prune_test.dev01");
+ try {
+ Assert.assertTrue(viewCache.addView(database, "view5", false, 0, viewPath));
+
+ Assert.assertTrue(viewCache.removeView(database, "view5"));
+
+ // the path of the removed view is not held by the trie any more
+ Assert.assertNull(getRootNode().getChild(database));
+ Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty());
+ } finally {
+ viewCache.removeView(database, "view5");
+ }
+ }
+
+ @Test
+ public void testRemoveViewOfAnotherDatabase() throws Exception {
+ final String database = "database_test";
+ final PartialPath viewPath = new PartialPath("root.database_test.dev01.**");
+ final PartialPath devicePath = new PartialPath("root.database_test.dev01");
+ try {
+ Assert.assertTrue(viewCache.addView(database, "view6", false, 0, viewPath));
+
+ // the view of another database is another view, even if the name and the path are the same
+ Assert.assertFalse(viewCache.removeView("another_database", "view6"));
+ Assert.assertTrue(isCovered(devicePath, "view6"));
+ } finally {
+ viewCache.removeView(database, "view6");
+ }
+ }
+
+ private boolean isCovered(final PartialPath devicePath, final String tableName) {
+ return viewCache.getTreeToTableViews(devicePath).stream()
+ .anyMatch(view -> tableName.equals(view.getTableName()));
+ }
+
+ private static IMemMNode getRootNode() throws Exception {
+ final Field field = TreeToTableViewCache.class.getDeclaredField("rootNode");
+ field.setAccessible(true);
+ return (IMemMNode) field.get(TreeToTableViewCache.getInstance());
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index f7a594659e7..a70315cca34 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -177,7 +177,8 @@ public class DataRegionTest {
DataNodeTableCache.getInstance()
.preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(), null);
DataNodeTableCache.getInstance()
- .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null);
+ .commitUpdateTable(
+ dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null, null);
}
@After
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java
index e9ea0025250..9aa2143ab8e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java
@@ -188,8 +188,11 @@ public class CompactionDataTypeAlterTableTest extends AbstractCompactionAlterDat
tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING));
tsTable.addColumnSchema(
new FieldColumnSchema("s1", dataType, TSEncoding.PLAIN, CompressionType.LZ4));
+ DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null);
+ DataNodeTableCache.getInstance()
+ .commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null, null);
DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null);
}
private TsFileResource generateInt32TableFile(TimeRange timeRange, boolean seq)
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java
index 877756b73d5..e372b0b43ac 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java
@@ -170,6 +170,6 @@ public class TableModelCompactionWithTTLTest extends AbstractTableModelCompactio
new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4));
tsTable.addProp(TsTable.TTL_PROPERTY, ttl + "");
DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null);
- DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null);
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java
index 49f22af8640..8d43c77970c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java
@@ -190,7 +190,7 @@ public class UnsealedTsFileRecoverPerformerTest {
testTable1.addColumnSchema(
new FieldColumnSchema("m1", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP));
DataNodeTableCache.getInstance().preUpdateTable(SG_NAME, testTable1, null);
- DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null);
+ DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null, null);
// generate crashed .tsfile
File file = new File(FILE_NAME);
Files.createDirectories(file.getParentFile().toPath());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java
index 306f3050c48..602ab575f6e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java
@@ -87,7 +87,7 @@ public class LoadTsFileTableSchemaTest {
table.addColumnSchema(
new FieldColumnSchema("value", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.LZ4));
DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null);
- DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null);
+ DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null, null);
}
@After
diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java
index 58e7f6fe220..966512e4bfb 100644
--- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java
@@ -38,6 +38,8 @@ public final class SchemaMessages {
public static final String CANNOT_REMOVE_TAG_COLUMN = "Cannot remove an tag column: %s";
public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE =
"Unknown table update operation type%s";
+ public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F =
+ "Unknown tree to table view operation type: %s";
// Column messages
public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE =
diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java
index 776690108a0..dcbdec3a1b6 100644
--- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java
@@ -38,6 +38,8 @@ public final class SchemaMessages {
public static final String CANNOT_REMOVE_TAG_COLUMN = "无法移除 tag 列:%s";
public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE =
"未知的 table 更新操作类型%s";
+ public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F =
+ "未知的树转表视图操作类型:%s";
// Column 相关消息
public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE =
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java
new file mode 100644
index 00000000000..c8e8855bd85
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.schema.table;
+
+import org.apache.iotdb.commons.i18n.SchemaMessages;
+
+public enum TreeToTableViewTrieOperationType {
+ /** Register the view, or replace the entry stored for the same view. */
+ ADD((byte) 0),
+ /** Unregister the view. */
+ REMOVE((byte) 1),
+
+ /** Replace the registered entry of the view with the one computed from the new table. */
+ ALTER_TAG((byte) 2),
+
+ ALTER_RENAME((byte) 3),
+
+ ALTER_AUTO_SYNC((byte) 4);
+
+ private final byte operationType;
+
+ TreeToTableViewTrieOperationType(final byte operationType) {
+ this.operationType = operationType;
+ }
+
+ public byte getOperationType() {
+ return operationType;
+ }
+
+ public static TreeToTableViewTrieOperationType getType(final byte type) {
+ switch (type) {
+ case -1:
+ return null;
+ case 0:
+ return ADD;
+ case 1:
+ return REMOVE;
+ case 2:
+ return ALTER_TAG;
+ case 3:
+ return ALTER_RENAME;
+ case 4:
+ return ALTER_AUTO_SYNC;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ SchemaMessages.EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F,
+ type));
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java
index 2acbb993254..e1068a21d15 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java
@@ -27,7 +27,9 @@ import org.apache.iotdb.commons.path.PathPatternUtil;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.iotdb.rpc.TSStatusCode;
+import java.util.HashSet;
import java.util.Objects;
+import java.util.Set;
public class TreeViewSchema {
public static final String ORIGINAL_NAME = "__original_name";
@@ -83,6 +85,23 @@ public class TreeViewSchema {
schema.getProps().put(ORIGINAL_NAME, name);
}
+ /**
+ * A source measurement can only be referenced once, so this set decides whether a column may
+ * still be added for one.
+ */
+ public static Set<String> getSourceNames(final TsTable view) {
+ final Set<String> sourceNames = new HashSet<>();
+ for (final TsTableColumnSchema schema : view.getColumnList()) {
+ if (Objects.isNull(schema.getProps())) {
+ // the columns which carry no props reference their own name
+ sourceNames.add(schema.getColumnName());
+ } else {
+ sourceNames.add(getSourceName(schema));
+ }
+ }
+ return sourceNames;
+ }
+
public static boolean isRestrict(final TsTable table) {
return table.getPropValue(RESTRICT).isPresent();
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java
index 681c638bc56..402045012e4 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java
@@ -68,6 +68,7 @@ public class TsTable {
// No column property key is supported yet.
public static final Set<String> COLUMN_ALLOWED_PROPERTIES = Collections.emptySet();
public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache";
+ public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema";
public static final Set<String> TABLE_ALLOWED_PROPERTIES =
Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY)));
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index 0b733829d87..a635bf0db9b 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -1368,6 +1368,23 @@ struct TCreateTableViewReq {
2: required bool replace
}
+ struct TAutoSyncViewColumnsFailure {
+ 1: required string database
+ 2: required string tableName
+ 3: required common.TSStatus status
+ }
+
+
+struct TAutoSyncViewColumnsResp{
+ 1: required common.TSStatus status
+ 2: optional list<TAutoSyncViewColumnsFailure> failedViews
+}
+
+struct TAutoSyncViewColumnsReq{
+ 1:required list<TAlterOrDropTableReq> viewInfos
+}
+
+
service IConfigNodeRPCService {
// ======================================================
@@ -2171,4 +2188,6 @@ service IConfigNodeRPCService {
// Table view
common.TSStatus createTableView(TCreateTableViewReq req)
+
+ TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req)
}
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index 76c0860a24f..57afa70e728 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -356,6 +356,14 @@ struct TUpdateTableReq {
1: required byte type
2: required binary tableInfo
3: optional string oldName
+ 4: optional TUpdateTree2TableView updateTree2TableViewInfo
+}
+
+struct TUpdateTree2TableView{
+ 1: required byte tree2TableOperationType
+ 2: optional string treePath
+ 3: optional bool isRestrict
+ 4: optional i32 tagNum
}
struct TInvalidateTableCacheReq {
@@ -450,6 +458,7 @@ struct TDeviceViewReq {
3: required i32 tagNumber
4: required bool restrict
5: optional set<string> requiredMeasurements
+ 6: optional bool tolerateTypeMismatch
}
struct TDeviceViewResp {