You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/11 16:55:53 UTC
[doris] 36/47: [enhancement](nereids)add eliminate filter on one row relation rule (#24980)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 369dd5d597e42f637ac06068e03e1f72b85f4ac5
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Wed Oct 11 16:12:24 2023 +0800
[enhancement](nereids)add eliminate filter on one row relation rule (#24980)
1.simplify PushdownFilterThroughSetOperation rule
2.add eliminate filter on one row relation rule
---
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/rewrite/EliminateFilter.java | 61 ++++++++++++++++++----
.../rewrite/PushdownFilterThroughSetOperation.java | 27 +---------
.../data/nereids_p0/test_filter_pushdown_set.out | 24 +++++++++
.../nereids_p0/test_filter_pushdown_set.groovy | 24 +++++++--
5 files changed, 98 insertions(+), 39 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 89de2182767..b1489e47165 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -194,6 +194,7 @@ public enum RuleType {
ELIMINATE_LIMIT_ON_ONE_ROW_RELATION(RuleTypeClass.REWRITE),
ELIMINATE_LIMIT_ON_EMPTY_RELATION(RuleTypeClass.REWRITE),
ELIMINATE_FILTER(RuleTypeClass.REWRITE),
+ ELIMINATE_FILTER_ON_ONE_RELATION(RuleTypeClass.REWRITE),
ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
index 5f5d8b39a83..2f46c43ecff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java
@@ -19,27 +19,36 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.util.ExpressionUtils;
-import com.google.common.collect.Sets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
-import java.util.Set;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
/**
* Eliminate filter which is FALSE or TRUE.
*/
-public class EliminateFilter extends OneRewriteRuleFactory {
+public class EliminateFilter implements RewriteRuleFactory {
@Override
- public Rule build() {
- return logicalFilter()
- .when(filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance))
+ public List<Rule> buildRules() {
+ return ImmutableList.of(logicalFilter().when(
+ filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance))
.thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
- Set<Expression> newConjuncts = Sets.newHashSetWithExpectedSize(filter.getConjuncts().size());
+ ImmutableSet.Builder newConjuncts = ImmutableSet.builder();
for (Expression expression : filter.getConjuncts()) {
if (expression == BooleanLiteral.FALSE) {
return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(),
@@ -48,12 +57,44 @@ public class EliminateFilter extends OneRewriteRuleFactory {
newConjuncts.add(expression);
}
}
- if (newConjuncts.isEmpty()) {
+
+ ImmutableSet<Expression> conjuncts = newConjuncts.build();
+ if (conjuncts.isEmpty()) {
+ return filter.child();
+ } else {
+ return new LogicalFilter<>(conjuncts, filter.child());
+ }
+ })
+ .toRule(RuleType.ELIMINATE_FILTER),
+ logicalFilter(logicalOneRowRelation()).thenApply(ctx -> {
+ LogicalFilter<LogicalOneRowRelation> filter = ctx.root;
+ Map<Expression, Expression> replaceMap =
+ filter.child().getOutputs().stream().filter(e -> e instanceof Alias)
+ .collect(Collectors.toMap(NamedExpression::toSlot, e -> ((Alias) e).child()));
+
+ ImmutableSet.Builder newConjuncts = ImmutableSet.builder();
+ ExpressionRewriteContext context =
+ new ExpressionRewriteContext(ctx.cascadesContext);
+ for (Expression expression : filter.getConjuncts()) {
+ Expression newExpr = ExpressionUtils.replace(expression, replaceMap);
+ Expression foldExpression =
+ FoldConstantRule.INSTANCE.rewrite(newExpr, context);
+
+ if (foldExpression == BooleanLiteral.FALSE) {
+ return new LogicalEmptyRelation(
+ ctx.statementContext.getNextRelationId(), filter.getOutput());
+ } else if (foldExpression != BooleanLiteral.TRUE) {
+ newConjuncts.add(expression);
+ }
+ }
+
+ ImmutableSet<Expression> conjuncts = newConjuncts.build();
+ if (conjuncts.isEmpty()) {
return filter.child();
} else {
- return new LogicalFilter<>(newConjuncts, filter.child());
+ return new LogicalFilter<>(conjuncts, filter.child());
}
})
- .toRule(RuleType.ELIMINATE_FILTER);
+ .toRule(RuleType.ELIMINATE_FILTER_ON_ONE_RELATION));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java
index 19f5243fc95..6b78ab1f367 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java
@@ -22,11 +22,8 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
-import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
-import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableSet;
@@ -44,25 +41,10 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalFilter(logicalSetOperation()).when(f -> f.child().getQualifier() == Qualifier.ALL).then(f -> {
+ return logicalFilter(logicalSetOperation()).then(f -> {
LogicalSetOperation setOperation = f.child();
-
- if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) {
- return f;
- }
-
List<Plan> newChildren = new ArrayList<>();
- boolean allOneRowRelation = true;
- boolean hasOneRowRelation = false;
for (Plan child : setOperation.children()) {
- if (child instanceof OneRowRelation) {
- // We shouldn't push down the 'filter' to 'oneRowRelation'.
- hasOneRowRelation = true;
- newChildren.add(child);
- continue;
- } else {
- allOneRowRelation = false;
- }
Map<Expression, Expression> replaceMap = new HashMap<>();
for (int i = 0; i < setOperation.getOutputs().size(); ++i) {
NamedExpression output = setOperation.getOutputs().get(i);
@@ -73,14 +55,7 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory {
ExpressionUtils.replace(conjunct, replaceMap)).collect(ImmutableSet.toImmutableSet());
newChildren.add(new LogicalFilter<>(newFilterPredicates, child));
}
- if (allOneRowRelation) {
- return f;
- }
- if (hasOneRowRelation) {
- // If there are some `OneRowRelation` exists, we need to keep the `filter`.
- return f.withChildren(((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren));
- }
return setOperation.withChildren(newChildren);
}).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION);
}
diff --git a/regression-test/data/nereids_p0/test_filter_pushdown_set.out b/regression-test/data/nereids_p0/test_filter_pushdown_set.out
index 42ecbc5ad88..6f6a6062fef 100644
--- a/regression-test/data/nereids_p0/test_filter_pushdown_set.out
+++ b/regression-test/data/nereids_p0/test_filter_pushdown_set.out
@@ -1,10 +1,34 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select1 --
1 2
+1 3
+1 3
-- !select2 --
1 2
+1 3
-- !select3 --
1 2
+-- !select4 --
+1 3
+
+-- !select5 --
+1 2
+1 2
+
+-- !select6 --
+1 2
+
+-- !select7 --
+1 2
+
+-- !select8 --
+
+-- !select22 --
+1 2
+
+-- !select23 --
+1 2
+
diff --git a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy
index 03c1b257981..7d25aade293 100644
--- a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy
+++ b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy
@@ -18,8 +18,26 @@
suite("test_filter_pushdown_set") {
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
- qt_select1 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;'
+ qt_select1 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 3) t where a = 1 order by a, b;'
+ qt_select2 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 3) t where a = 1 order by a, b;'
+ qt_select3 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1 order by a, b;'
+ qt_select4 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where a = 1 order by a, b;'
+ qt_select5 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 2) t where b = 2 order by a, b;'
+ qt_select6 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 2) t where b = 2 order by a, b;'
+ qt_select7 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where b = 2 order by a, b;'
+ qt_select8 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2 order by a, b;'
+
+ explain {
+ sql("select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1;")
+ notContains "VSELECT"
+ }
+
+ explain {
+ sql("select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2;")
+ notContains "1 | 3"
+ }
+
sql "SET enable_nereids_planner=false"
- qt_select2 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;'
- qt_select3 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1;'
+ qt_select22 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1 order by a, b;'
+ qt_select23 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1 order by a, b;'
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org