You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/09 04:10:59 UTC
[doris] 23/29: [fix](nereids) add push down filter and project through cte anchor rules (#20547)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 66e2d9c877531433a8eb77b3955bd833311a3ec0
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Thu Jun 8 10:34:42 2023 +0800
[fix](nereids) add push down filter and project through cte anchor rules (#20547)
we should not plan any Filter or Project above CteAnchor, because there are project or filter under anchor sometimes.
and the whole plan can not translate to a valid plan for BE.
---
.../org/apache/doris/nereids/rules/RuleSet.java | 6 +++-
.../org/apache/doris/nereids/rules/RuleType.java | 2 ++
.../logical/PushdownFilterThroughCTEAnchor.java | 40 ++++++++++++++++++++++
.../logical/PushdownProjectThroughCTEAnchor.java | 40 ++++++++++++++++++++++
.../data/nereids_p0/aggregate/aggregate_count1.out | 3 ++
.../nereids_p0/aggregate/aggregate_count1.groovy | 28 +++++++++++++++
6 files changed, 118 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 2763c9ce9c..84f7479fa6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -76,6 +76,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownExpressionsInHashCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggregation;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughCTE;
+import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughCTEAnchor;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughRepeat;
@@ -84,6 +85,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughCTE;
+import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughCTEAnchor;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit;
import com.google.common.collect.ImmutableList;
@@ -131,7 +133,9 @@ public class RuleSet {
new MergeGenerates(),
new MergeLimits(),
new PushdownFilterThroughCTE(),
- new PushdownProjectThroughCTE());
+ new PushdownProjectThroughCTE(),
+ new PushdownFilterThroughCTEAnchor(),
+ new PushdownProjectThroughCTEAnchor());
public static final List<Rule> IMPLEMENTATION_RULES = planRuleFactories()
.add(new LogicalCTEProduceToPhysicalCTEProduce())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index fbe7deaae4..2c0e6479c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -142,6 +142,7 @@ public enum RuleType {
PUSHDOWN_FILTER_THROUGH_SORT(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_CTE(RuleTypeClass.REWRITE),
+ PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
@@ -231,6 +232,7 @@ public enum RuleType {
COLLECT_PROJECT_ABOVE_FILTER_CONSUMER(RuleTypeClass.REWRITE),
CTE_PRODUCER_REWRITE(RuleTypeClass.REWRITE),
PUSH_DOWN_PROJECT_THROUGH_CTE(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE),
INLINE_CTE(RuleTypeClass.REWRITE),
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java
new file mode 100644
index 0000000000..25adf9a576
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughCTEAnchor.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+
+/**
+ * Push filter through CTEAnchor.
+ */
+public class PushdownFilterThroughCTEAnchor extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalFilter(logicalCTEAnchor()).thenApply(ctx -> {
+ LogicalFilter<LogicalCTEAnchor<Plan, Plan>> filter = ctx.root;
+ LogicalCTEAnchor<Plan, Plan> anchor = filter.child();
+ return anchor.withChildren(anchor.left(), filter.withChildren((Plan) anchor.right()));
+ }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java
new file mode 100644
index 0000000000..1d6d64529a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughCTEAnchor.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+/**
+ * Push project through CTEAnchor.
+ */
+public class PushdownProjectThroughCTEAnchor extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalProject(logicalCTEAnchor()).thenApply(ctx -> {
+ LogicalProject<LogicalCTEAnchor<Plan, Plan>> project = ctx.root;
+ LogicalCTEAnchor<Plan, Plan> anchor = project.child();
+ return anchor.withChildren(anchor.child(0), project.withChildren(anchor.child(1)));
+ }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR);
+ }
+}
diff --git a/regression-test/data/nereids_p0/aggregate/aggregate_count1.out b/regression-test/data/nereids_p0/aggregate/aggregate_count1.out
index 84f6fe8329..57516fb642 100644
--- a/regression-test/data/nereids_p0/aggregate/aggregate_count1.out
+++ b/regression-test/data/nereids_p0/aggregate/aggregate_count1.out
@@ -2,3 +2,6 @@
-- !select --
8
+-- !select2 --
+8
+
diff --git a/regression-test/suites/nereids_p0/aggregate/aggregate_count1.groovy b/regression-test/suites/nereids_p0/aggregate/aggregate_count1.groovy
index 0bc1fb0865..6cc2d417e8 100644
--- a/regression-test/suites/nereids_p0/aggregate/aggregate_count1.groovy
+++ b/regression-test/suites/nereids_p0/aggregate/aggregate_count1.groovy
@@ -73,5 +73,33 @@ suite("aggregate_count1", "query") {
(SELECT virtuleUniqKey as max_virtuleUniqKey FROM t1 ORDER BY proportion DESC LIMIT 1 ) tableWithMaxId
ORDER BY identityCode) t_a76fe3e829ddb51;
"""
+ sql """set experimental_enable_pipeline_engine=true"""
+ qt_select2 """ SELECT count(1) FROM (WITH t1 AS (
+ WITH t AS (
+ SELECT * FROM aggregate_count1
+ )
+ SELECT
+ identityCode,
+ COUNT(1) as dataAmount,
+ ROUND(COUNT(1) / tableWithSum.sumResult,4) as proportion,
+ MD5(identityCode) as virtuleUniqKey
+ FROM t,(SELECT COUNT(1) as sumResult from t) tableWithSum
+ GROUP BY identityCode ,tableWithSum.sumResult
+ )
+ SELECT
+ identityCode,dataAmount,
+ (
+ CASE
+ WHEN t1.virtuleUniqKey = tableWithMaxId.max_virtuleUniqKey THEN
+ ROUND(proportion + calcTheTail, 4)
+ ELSE
+ proportion
+ END
+ ) proportion
+ FROM t1,
+ (SELECT (1 - sum(t1.proportion)) as calcTheTail FROM t1 ) tableWithTail,
+ (SELECT virtuleUniqKey as max_virtuleUniqKey FROM t1 ORDER BY proportion DESC LIMIT 1 ) tableWithMaxId
+ ORDER BY identityCode) t_a76fe3e829ddb51;
+ """
sql "drop table aggregate_count1"
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org