You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/09/26 08:03:06 UTC

(iotdb) branch pipe-table-model-3 updated (5f7b86eb35c -> 73827d0d09c)

This is an automated email from the ASF dual-hosted git repository.

rong pushed a change to branch pipe-table-model-3
in repository https://gitbox.apache.org/repos/asf/iotdb.git


    from 5f7b86eb35c IoTDBDataRegionTableModelExtractor
     new 8cadd0f25f4 PipeDataRegionExtractorConstructor impl
     add 01370932799 Feature/show ainodes command (#13592)
     add 99037029548 Fix the error in aggregation merge sort when device locates in multi data regions
     add 734786f09e4 Fixed database names in "show regions" in table model
     add 574bbd1b2ba Feature/dependency check (#13587)
     add 374deefeb86 Update ConfigPhysicalReadPlan.java (#13624)
     add c4f17560624 Pipe: Fixed the bug that different types in schema snapshot transfer may cause transfer failure (#13627)
     add e7a37dc03ab Pipe: Fixed the bug that alter source will not alter the reference count of ConfigRegionListeningQueue (#13626)
     add aa2fc4c326c [RatisConsensus] Bump ratis version to 3.1.1 #13629
     new 73827d0d09c Merge branch 'master' of https://github.com/apache/iotdb into pipe-table-model-3

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../{unit-test.yml => dependency-check.yml}        |  21 +--
 dependencies.json                                  | 167 +++++++++++++++++++
 .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java |  20 ++-
 iotdb-core/ainode/pyproject.toml                   |   2 +-
 .../request/read/ConfigPhysicalReadPlan.java       |   4 +-
 .../agent/runtime/PipeConfigNodeRuntimeAgent.java  |  13 +-
 .../agent/runtime/PipeConfigRegionListener.java    |   7 +-
 .../confignode/persistence/pipe/PipeInfo.java      |  33 +++-
 .../schemaregion/SchemaExecutionVisitor.java       |  12 ++
 .../PipeDataRegionExtractorConstructor.java        |  48 ++++++
 .../process/AggregationMergeSortOperator.java      |   3 +-
 .../iotdb/db/queryengine/plan/Coordinator.java     |   2 +
 .../execution/config/TableConfigTaskVisitor.java   |  18 ++-
 .../execution/config/TreeConfigTaskVisitor.java    |  16 +-
 .../config/executor/ClusterConfigTaskExecutor.java | 113 ++++---------
 .../config/executor/IConfigTaskExecutor.java       |  20 +--
 .../metadata/{model => }/ShowAINodesTask.java      |  11 +-
 .../config/metadata/ShowDataNodesTask.java         |   2 +-
 .../execution/config/metadata/ShowRegionTask.java  |  29 ++--
 ...ShowDataNodesTask.java => ShowAINodesTask.java} |  11 +-
 .../metadata/relational/ShowConfigNodesTask.java   |   9 +-
 .../metadata/relational/ShowDataNodesTask.java     |   9 +-
 .../plan/planner/OperatorTreeGenerator.java        |   2 +-
 .../plan/relational/sql/ast/AstVisitor.java        |   4 +
 .../ast/{ShowDataNodes.java => ShowAINodes.java}   |   8 +-
 .../plan/relational/sql/parser/AstBuilder.java     |   6 +
 .../operator/AggregationMergeSortOperatorTest.java | 177 +++++++++++++++++++++
 .../constructor/PipeExtractorConstructor.java      |   2 +-
 .../plugin/constructor/PipePluginConstructor.java  |   2 +-
 .../db/relational/grammar/sql/RelationalSql.g4     |   6 +
 pom.xml                                            |  87 +++++++++-
 src/main/groovy/checkDependencies.groovy           |  60 +++++++
 .../main/xslt/sbom-filter.xsl                      |  35 ++--
 33 files changed, 742 insertions(+), 217 deletions(-)
 copy .github/workflows/{unit-test.yml => dependency-check.yml} (60%)
 create mode 100644 dependencies.json
 rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/{model => }/ShowAINodesTask.java (90%)
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{ShowDataNodesTask.java => ShowAINodesTask.java} (91%)
 copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{ShowDataNodes.java => ShowAINodes.java} (90%)
 create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationMergeSortOperatorTest.java
 create mode 100644 src/main/groovy/checkDependencies.groovy
 copy iotdb-core/confignode/src/assembly/resources/conf/logback-tool.xml => src/main/xslt/sbom-filter.xsl (57%)


(iotdb) 01/02: PipeDataRegionExtractorConstructor impl

Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch pipe-table-model-3
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 8cadd0f25f49f32e170b37a17b524116dc9c12a3
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu Sep 26 16:02:34 2024 +0800

    PipeDataRegionExtractorConstructor impl
---
 .../PipeDataRegionExtractorConstructor.java        | 48 ++++++++++++++++++++++
 .../constructor/PipeExtractorConstructor.java      |  2 +-
 .../plugin/constructor/PipePluginConstructor.java  |  2 +-
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionExtractorConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionExtractorConstructor.java
index ddf6cdb6b8b..46636d76592 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionExtractorConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionExtractorConstructor.java
@@ -23,10 +23,24 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
 import org.apache.iotdb.commons.pipe.agent.plugin.builtin.extractor.donothing.DoNothingExtractor;
 import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeExtractorConstructor;
 import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
+import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
 import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
+import org.apache.iotdb.db.pipe.table.extractor.dataregion.IoTDBDataRegionTableModelExtractor;
+import org.apache.iotdb.pipe.api.PipeExtractor;
+import org.apache.iotdb.pipe.api.PipePlugin;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Supplier;
 
 class PipeDataRegionExtractorConstructor extends PipeExtractorConstructor {
 
+  // TODO: consider refactor the plugin constructors to a more generic way
+  protected final Map<String, Supplier<PipePlugin>> tableModelPluginConstructors = new HashMap<>();
+
   PipeDataRegionExtractorConstructor(DataNodePipePluginMetaKeeper pipePluginMetaKeeper) {
     super(pipePluginMetaKeeper);
   }
@@ -42,5 +56,39 @@ class PipeDataRegionExtractorConstructor extends PipeExtractorConstructor {
         BuiltinPipePlugin.DO_NOTHING_SOURCE.getPipePluginName(), DoNothingExtractor::new);
     pluginConstructors.put(
         BuiltinPipePlugin.IOTDB_SOURCE.getPipePluginName(), IoTDBDataRegionExtractor::new);
+
+    // Plugins for table model only
+    tableModelPluginConstructors.put(
+        BuiltinPipePlugin.DO_NOTHING_EXTRACTOR.getPipePluginName(), DoNothingExtractor::new);
+    tableModelPluginConstructors.put(
+        BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName(),
+        IoTDBDataRegionTableModelExtractor::new);
+
+    tableModelPluginConstructors.put(
+        BuiltinPipePlugin.DO_NOTHING_SOURCE.getPipePluginName(), DoNothingExtractor::new);
+    tableModelPluginConstructors.put(
+        BuiltinPipePlugin.IOTDB_SOURCE.getPipePluginName(),
+        IoTDBDataRegionTableModelExtractor::new);
+  }
+
+  @Override
+  public final PipeExtractor reflectPlugin(PipeParameters extractorParameters) {
+    return extractorParameters
+            .getStringOrDefault(
+                SystemConstant.SESSION_MODEL_KEY, SystemConstant.SESSION_MODEL_TREE_VALUE)
+            .equals(SystemConstant.SESSION_MODEL_TREE_VALUE)
+        ? super.reflectPlugin(extractorParameters)
+        : reflectTableModelPlugin(
+            extractorParameters
+                .getStringOrDefault(
+                    Arrays.asList(
+                        PipeExtractorConstant.EXTRACTOR_KEY, PipeExtractorConstant.SOURCE_KEY),
+                    BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName())
+                .toLowerCase());
+  }
+
+  private PipeExtractor reflectTableModelPlugin(String pluginName) {
+    return (PipeExtractor)
+        tableModelPluginConstructors.getOrDefault(pluginName, () -> reflect(pluginName)).get();
   }
 }
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipeExtractorConstructor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipeExtractorConstructor.java
index 001ae49e939..03d6836aea6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipeExtractorConstructor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipeExtractorConstructor.java
@@ -38,7 +38,7 @@ public abstract class PipeExtractorConstructor extends PipePluginConstructor {
   }
 
   @Override
-  public final PipeExtractor reflectPlugin(PipeParameters extractorParameters) {
+  public PipeExtractor reflectPlugin(PipeParameters extractorParameters) {
     return (PipeExtractor)
         reflectPluginByKey(
             extractorParameters
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipePluginConstructor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipePluginConstructor.java
index cbb25340f00..e311471f450 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipePluginConstructor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/constructor/PipePluginConstructor.java
@@ -60,7 +60,7 @@ public abstract class PipePluginConstructor {
     return pluginConstructors.getOrDefault(pluginKey, () -> reflect(pluginKey)).get();
   }
 
-  private PipePlugin reflect(String pluginName) {
+  protected PipePlugin reflect(String pluginName) {
     if (pluginMetaKeeper == null) {
       throw new PipeException(
           "Failed to reflect PipePlugin instance, because PipePluginMetaKeeper is null.");


(iotdb) 02/02: Merge branch 'master' of https://github.com/apache/iotdb into pipe-table-model-3

Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch pipe-table-model-3
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 73827d0d09c2f2af44daeab12163f436ac28574d
Merge: 8cadd0f25f4 aa2fc4c326c
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu Sep 26 16:02:52 2024 +0800

    Merge branch 'master' of https://github.com/apache/iotdb into pipe-table-model-3

 .github/workflows/dependency-check.yml             |  59 +++++++
 dependencies.json                                  | 167 +++++++++++++++++++
 .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java |  20 ++-
 iotdb-core/ainode/pyproject.toml                   |   2 +-
 .../request/read/ConfigPhysicalReadPlan.java       |   4 +-
 .../agent/runtime/PipeConfigNodeRuntimeAgent.java  |  13 +-
 .../agent/runtime/PipeConfigRegionListener.java    |   7 +-
 .../confignode/persistence/pipe/PipeInfo.java      |  33 +++-
 .../schemaregion/SchemaExecutionVisitor.java       |  12 ++
 .../process/AggregationMergeSortOperator.java      |   3 +-
 .../iotdb/db/queryengine/plan/Coordinator.java     |   2 +
 .../execution/config/TableConfigTaskVisitor.java   |  18 ++-
 .../execution/config/TreeConfigTaskVisitor.java    |  16 +-
 .../config/executor/ClusterConfigTaskExecutor.java | 113 ++++---------
 .../config/executor/IConfigTaskExecutor.java       |  20 +--
 .../metadata/{model => }/ShowAINodesTask.java      |  11 +-
 .../config/metadata/ShowDataNodesTask.java         |   2 +-
 .../execution/config/metadata/ShowRegionTask.java  |  29 ++--
 ...ShowDataNodesTask.java => ShowAINodesTask.java} |  11 +-
 .../metadata/relational/ShowConfigNodesTask.java   |   9 +-
 .../metadata/relational/ShowDataNodesTask.java     |   9 +-
 .../plan/planner/OperatorTreeGenerator.java        |   2 +-
 .../plan/relational/sql/ast/AstVisitor.java        |   4 +
 .../plan/relational/sql/ast/ShowAINodes.java       |  66 ++++++++
 .../plan/relational/sql/parser/AstBuilder.java     |   6 +
 .../operator/AggregationMergeSortOperatorTest.java | 177 +++++++++++++++++++++
 .../db/relational/grammar/sql/RelationalSql.g4     |   6 +
 pom.xml                                            |  87 +++++++++-
 src/main/groovy/checkDependencies.groovy           |  60 +++++++
 src/main/xslt/sbom-filter.xsl                      |  41 +++++
 30 files changed, 828 insertions(+), 181 deletions(-)