You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/07 02:36:38 UTC
[doris] branch master updated: [fix](nereids) filter and project node should be pushed down through cte (#20508)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cd70c37402 [fix](nereids) filter and project node should be pushed down through cte (#20508)
cd70c37402 is described below
commit cd70c37402654851248b891d318409f185bb7d3c
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Wed Jun 7 10:36:32 2023 +0800
[fix](nereids) filter and project node should be pushed down through cte (#20508)
1.move PushdownFilterThroughCTEAnchor and PushdownProjectThroughCTEAnchor into PUSH_DOWN_FILTERS rule set
2.move PushdownFilterThroughProject before MergeProjectPostProcessor
---
.../doris/nereids/jobs/batch/NereidsRewriter.java | 6 ----
.../nereids/processor/post/PlanPostProcessors.java | 2 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 6 ++--
.../org/apache/doris/nereids/rules/RuleType.java | 4 +--
...TEAnchor.java => PushdownFilterThroughCTE.java} | 16 +++++-----
...EAnchor.java => PushdownProjectThroughCTE.java} | 16 +++++-----
regression-test/data/nereids_syntax_p0/cte.out | 10 +++++++
.../suites/nereids_syntax_p0/cte.groovy | 34 ++++++++++++++++++++++
8 files changed, 67 insertions(+), 27 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
index 8122f2669a..35689b9e5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
@@ -72,11 +72,9 @@ import org.apache.doris.nereids.rules.rewrite.logical.PruneFileScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet;
import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin;
-import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughCTEAnchor;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownLimit;
-import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughCTEAnchor;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownTopNThroughWindow;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
import org.apache.doris.nereids.rules.rewrite.logical.SemiJoinCommute;
@@ -127,10 +125,6 @@ public class NereidsRewriter extends BatchRewriteJob {
)
),
- topic("Rewrite CTE", topDown(
- new PushdownFilterThroughCTEAnchor(),
- new PushdownProjectThroughCTEAnchor())),
-
topic("Subquery unnesting",
custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION, AggScalarSubQueryToWindowFunction::new),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
index 4e72b8738d..c7fe4309e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
@@ -58,8 +58,8 @@ public class PlanPostProcessors {
public List<PlanPostProcessor> getProcessors() {
// add processor if we need
Builder<PlanPostProcessor> builder = ImmutableList.builder();
- builder.add(new MergeProjectPostProcessor());
builder.add(new PushdownFilterThroughProject());
+ builder.add(new MergeProjectPostProcessor());
builder.add(new FragmentProcessor());
if (!cascadesContext.getConnectContext().getSessionVariable().getRuntimeFilterMode()
.toUpperCase().equals(TRuntimeFilterMode.OFF.name())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 049680ee3f..2763c9ce9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -75,7 +75,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownExpressionsInHashCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggregation;
-import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughCTEAnchor;
+import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughCTE;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughRepeat;
@@ -83,6 +83,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSetOp
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownJoinOtherCondition;
+import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughCTE;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit;
import com.google.common.collect.ImmutableList;
@@ -129,7 +130,8 @@ public class RuleSet {
new MergeFilters(),
new MergeGenerates(),
new MergeLimits(),
- new PushdownFilterThroughCTEAnchor());
+ new PushdownFilterThroughCTE(),
+ new PushdownProjectThroughCTE());
public static final List<Rule> IMPLEMENTATION_RULES = planRuleFactories()
.add(new LogicalCTEProduceToPhysicalCTEProduce())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 6bce357c7a..fbe7deaae4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -141,7 +141,7 @@ public enum RuleType {
PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SORT(RuleTypeClass.REWRITE),
- PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE),
+ PUSHDOWN_FILTER_THROUGH_CTE(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
@@ -230,7 +230,7 @@ public enum RuleType {
COLLECT_PROJECT_ABOVE_FILTER_CONSUMER(RuleTypeClass.REWRITE),
CTE_PRODUCER_REWRITE(RuleTypeClass.REWRITE),
- PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_CTE(RuleTypeClass.REWRITE),
INLINE_CTE(RuleTypeClass.REWRITE),
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTE.java
similarity index 68%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTE.java
index 25adf9a576..09f9c32db7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTE.java
@@ -21,20 +21,20 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
/**
- * Push filter through CTEAnchor.
+ * Push filter through CTE.
*/
-public class PushdownFilterThroughCTEAnchor extends OneRewriteRuleFactory {
+public class PushdownFilterThroughCTE extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalFilter(logicalCTEAnchor()).thenApply(ctx -> {
- LogicalFilter<LogicalCTEAnchor<Plan, Plan>> filter = ctx.root;
- LogicalCTEAnchor<Plan, Plan> anchor = filter.child();
- return anchor.withChildren(anchor.left(), filter.withChildren((Plan) anchor.right()));
- }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR);
+ return logicalFilter(logicalCTE()).thenApply(ctx -> {
+ LogicalFilter<LogicalCTE<Plan>> filter = ctx.root;
+ LogicalCTE<Plan> anchor = filter.child();
+ return anchor.withChildren(filter.withChildren(anchor.child()));
+ }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_CTE);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTE.java
similarity index 68%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTE.java
index 1d6d64529a..8ff5df5c5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTE.java
@@ -21,20 +21,20 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
/**
- * Push project through CTEAnchor.
+ * Push project through CTE.
*/
-public class PushdownProjectThroughCTEAnchor extends OneRewriteRuleFactory {
+public class PushdownProjectThroughCTE extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalProject(logicalCTEAnchor()).thenApply(ctx -> {
- LogicalProject<LogicalCTEAnchor<Plan, Plan>> project = ctx.root;
- LogicalCTEAnchor<Plan, Plan> anchor = project.child();
- return anchor.withChildren(anchor.child(0), project.withChildren(anchor.child(1)));
- }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR);
+ return logicalProject(logicalCTE()).thenApply(ctx -> {
+ LogicalProject<LogicalCTE<Plan>> project = ctx.root;
+ LogicalCTE<Plan> anchor = project.child();
+ return anchor.withChildren(project.withChildren(anchor.child()));
+ }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_CTE);
}
}
diff --git a/regression-test/data/nereids_syntax_p0/cte.out b/regression-test/data/nereids_syntax_p0/cte.out
index 360e8fb37c..414dd7f9c1 100644
--- a/regression-test/data/nereids_syntax_p0/cte.out
+++ b/regression-test/data/nereids_syntax_p0/cte.out
@@ -72,3 +72,13 @@ ASIA 1
29
9
+-- !cte13 --
+9
+15
+29
+
+-- !cte14 --
+9
+15
+29
+
diff --git a/regression-test/suites/nereids_syntax_p0/cte.groovy b/regression-test/suites/nereids_syntax_p0/cte.groovy
index 9d847fffd0..d2b6eefa0d 100644
--- a/regression-test/suites/nereids_syntax_p0/cte.groovy
+++ b/regression-test/suites/nereids_syntax_p0/cte.groovy
@@ -237,6 +237,40 @@ suite("cte") {
"""
+ qt_cte13 """
+ SELECT abs(dd.s_suppkey)
+ FROM (
+ WITH part AS
+ (SELECT s_suppkey
+ FROM supplier
+ WHERE s_suppkey < 30 )
+ SELECT p1.s_suppkey
+ FROM part p1
+ JOIN part p2
+ ON p1.s_suppkey = p2.s_suppkey
+ WHERE p1.s_suppkey > 0 ) dd
+ WHERE dd.s_suppkey > 0
+ ORDER BY dd.s_suppkey;
+ """
+
+ sql "set experimental_enable_pipeline_engine=true"
+
+ qt_cte14 """
+ SELECT abs(dd.s_suppkey)
+ FROM (
+ WITH part AS
+ (SELECT s_suppkey
+ FROM supplier
+ WHERE s_suppkey < 30 )
+ SELECT p1.s_suppkey
+ FROM part p1
+ JOIN part p2
+ ON p1.s_suppkey = p2.s_suppkey
+ WHERE p1.s_suppkey > 0 ) dd
+ WHERE dd.s_suppkey > 0
+ ORDER BY dd.s_suppkey;
+ """
+
test {
sql = "WITH cte1 (a1, A1) AS (SELECT * FROM supplier) SELECT * FROM cte1"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org