You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/11/03 09:37:53 UTC
(doris) branch master updated: [opt](Nereids) Support to get notdeterministic functions and base tables from plan (#26120)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b19198b3052 [opt](Nereids) Support to get notdeterministic functions and base tables from plan (#26120)
b19198b3052 is described below
commit b19198b3052eaa43105f15c32817da894f9f1bc3
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Fri Nov 3 17:37:46 2023 +0800
[opt](Nereids) Support to get notdeterministic functions and base tables from plan (#26120)
Support to get notdeterministic functions and base tables from plan.
These are implemented by Plan Visitor
**table collect useage example as following:**
```
TableCollectorContext collectorContext =
new PlanVisitors.TableCollectorContext(Sets.newHashSet(TableType.OLAP));
physicalPlan.accept(PlanVisitors.TABLE_COLLECTOR, collectorContext);
```
`collectorContext.getCollectedTables()` has the tables
**nondeterministic functions collect useage example as following:**
```
List<Expression> functionCollectResult = new ArrayList<>();
plan.accept(PlanVisitors.NondeterministicCollector.INSTANCE, collectResult);
```
`functionCollectResult` has the nondeterministic functions
---
.../visitor/NondeterministicFunctionCollector.java | 47 +++++++
.../trees/plans/visitor/TableCollector.java | 69 ++++++++++
.../doris/nereids/trees/plans/PlanVisitorTest.java | 145 +++++++++++++++++++++
3 files changed, 261 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
new file mode 100644
index 00000000000..b17be42d383
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.visitor;
+
+import org.apache.doris.nereids.trees.TreeNode;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
+import org.apache.doris.nereids.trees.plans.Plan;
+
+import java.util.List;
+
+/**
+ * Collect the nondeterministic expr in plan, these expressions will be put into context
+ */
+public class NondeterministicFunctionCollector
+ extends DefaultPlanVisitor<Void, List<TreeNode<Expression>>> {
+
+ public static final NondeterministicFunctionCollector INSTANCE
+ = new NondeterministicFunctionCollector();
+
+ @Override
+ public Void visit(Plan plan, List<TreeNode<Expression>> collectedExpressions) {
+ List<? extends Expression> expressions = plan.getExpressions();
+ if (expressions.isEmpty()) {
+ return super.visit(plan, collectedExpressions);
+ }
+ expressions.forEach(expression -> {
+ collectedExpressions.addAll(expression.collect(Nondeterministic.class::isInstance));
+ });
+ return super.visit(plan, collectedExpressions);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
new file mode 100644
index 00000000000..54ffab75966
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.visitor;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Collect the table in plan
+ * Note: will not get table if table is eliminated by EmptyRelation in rewrite.
+ */
+public class TableCollector extends DefaultPlanVisitor<Void, TableCollectorContext> {
+
+ public static final TableCollector INSTANCE = new TableCollector();
+
+ @Override
+ public Void visit(Plan plan, TableCollectorContext context) {
+ if (plan instanceof CatalogRelation) {
+ TableIf table = ((CatalogRelation) plan).getTable();
+ if (context.getTargetTableTypes().contains(table.getType())) {
+ context.getCollectedTables().add(table);
+ }
+ }
+ return super.visit(plan, context);
+ }
+
+ /**
+ * The context for table collecting, it contains the target collect table types
+ * and the result of collect.
+ */
+ public static final class TableCollectorContext {
+ private final List<TableIf> collectedTables = new ArrayList<>();
+ private final Set<TableType> targetTableTypes;
+
+ public TableCollectorContext(Set<TableType> targetTableTypes) {
+ this.targetTableTypes = targetTableTypes;
+ }
+
+ public List<TableIf> getCollectedTables() {
+ return collectedTables;
+ }
+
+ public Set<TableType> getTargetTableTypes() {
+ return targetTableTypes;
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
new file mode 100644
index 00000000000..e70470c4682
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.nereids.trees.TreeNode;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentDate;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Now;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
+import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
+import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Tests for plan visitors to make sure the result meets expectation.
+ */
+public class PlanVisitorTest extends TestWithFeService {
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("visitor_test");
+ useDatabase("visitor_test");
+
+ createTable("CREATE TABLE `table1` (\n"
+ + " `c1` varchar(20) NULL,\n"
+ + " `c2` bigint(20) NULL,\n"
+ + " `c3` int(20) not NULL,\n"
+ + " `k4` bitmap BITMAP_UNION NULL,\n"
+ + " `k5` bitmap BITMAP_UNION NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "COMMENT 'OLAP'\n"
+ + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\" \n"
+ + ");");
+
+ createTable("CREATE TABLE `table2` (\n"
+ + " `c1` bigint(20) NULL,\n"
+ + " `c2` bigint(20) NULL,\n"
+ + " `c3` bigint(20) not NULL,\n"
+ + " `k4` bitmap BITMAP_UNION NULL,\n"
+ + " `k5` bitmap BITMAP_UNION NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "COMMENT 'OLAP'\n"
+ + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ");");
+
+ createView("CREATE VIEW `view1` AS SELECT t1.*, current_date() FROM\n"
+ + "`table1` t1 LEFT JOIN\n"
+ + "`table2` t2 ON t1.c1 = t2.c1;");
+ }
+
+ @Test
+ public void test1() {
+ PlanChecker.from(connectContext)
+ .checkPlannerResult("SELECT *, now() FROM table1 "
+ + "LEFT SEMI JOIN table2 ON table1.c1 = table2.c1 "
+ + "WHERE table1.c1 IN (SELECT c1 FROM table2) OR table1.c1 < 10",
+ nereidsPlanner -> {
+ PhysicalPlan physicalPlan = nereidsPlanner.getPhysicalPlan();
+ List<TreeNode<Expression>> collectResult = new ArrayList<>();
+ // Check nondeterministic collect
+ physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult);
+ Assertions.assertEquals(1, collectResult.size());
+ Assertions.assertTrue(collectResult.get(0) instanceof Now);
+ // Check get tables
+ TableCollectorContext collectorContext =
+ new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP));
+ physicalPlan.accept(TableCollector.INSTANCE, collectorContext);
+ Assertions.assertEquals(3, collectorContext.getCollectedTables().size());
+ List<String> expectedTables = new ArrayList<>();
+ expectedTables.add("table1");
+ expectedTables.add("table2");
+ expectedTables.add("table2");
+ Assertions.assertEquals(
+ collectorContext.getCollectedTables().stream()
+ .map(TableIf::getName)
+ .collect(Collectors.toList()),
+ expectedTables);
+ });
+ }
+
+ @Test
+ public void test2() {
+ PlanChecker.from(connectContext)
+ .checkPlannerResult("SELECT view1.*, now() FROM view1 "
+ + "LEFT SEMI JOIN table2 ON view1.c1 = table2.c1 "
+ + "WHERE view1.c1 IN (SELECT c1 FROM table2) OR view1.c1 < 10",
+ nereidsPlanner -> {
+ PhysicalPlan physicalPlan = nereidsPlanner.getPhysicalPlan();
+ List<TreeNode<Expression>> collectResult = new ArrayList<>();
+ // Check nondeterministic collect
+ physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult);
+ Assertions.assertEquals(2, collectResult.size());
+ Assertions.assertTrue(collectResult.get(0) instanceof Now);
+ Assertions.assertTrue(collectResult.get(1) instanceof CurrentDate);
+ // Check get tables
+ TableCollectorContext collectorContext =
+ new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP));
+ physicalPlan.accept(TableCollector.INSTANCE, collectorContext);
+ Assertions.assertEquals(4, collectorContext.getCollectedTables().size());
+ List<String> expectedTables = new ArrayList<>();
+ expectedTables.add("table1");
+ expectedTables.add("table2");
+ expectedTables.add("table2");
+ expectedTables.add("table2");
+ Assertions.assertEquals(
+ collectorContext.getCollectedTables().stream()
+ .map(TableIf::getName)
+ .collect(Collectors.toList()),
+ expectedTables);
+ });
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org