You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/06 15:15:11 UTC
[doris] 09/36: [fix](Nereids): fix filter can't be pushdown unionAll (#20310)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 31bd9ba03cc26818bd2d440702815c2b7956e8e4
Author: jakevin <ja...@gmail.com>
AuthorDate: Mon Jun 5 16:56:25 2023 +0800
[fix](Nereids): fix filter can't be pushdown unionAll (#20310)
---
.../logical/PushdownFilterThroughSetOperation.java | 21 ++---
.../nereids/trees/plans/logical/LogicalExcept.java | 5 -
.../trees/plans/logical/LogicalIntersect.java | 5 -
.../trees/plans/logical/LogicalSetOperation.java | 3 -
.../nereids/trees/plans/logical/LogicalUnion.java | 5 -
.../nereids_tpcds_shape_sf100_p0/shape/query75.out | 104 ++++++++++-----------
6 files changed, 58 insertions(+), 85 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java
index 218bf9e2ee..f678951d78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java
@@ -45,11 +45,11 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalFilter(logicalSetOperation()).then(filter -> {
- LogicalSetOperation setOperation = filter.child();
+ return logicalFilter(logicalSetOperation()).when(f -> f.child().getQualifier() == Qualifier.ALL).then(f -> {
+ LogicalSetOperation setOperation = f.child();
if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) {
- return filter;
+ return f;
}
List<Plan> newChildren = new ArrayList<>();
@@ -70,26 +70,19 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory {
replaceMap.put(output, child.getOutput().get(i));
}
- Set<Expression> newFilterPredicates = filter.getConjuncts().stream().map(conjunct ->
+ Set<Expression> newFilterPredicates = f.getConjuncts().stream().map(conjunct ->
ExpressionUtils.replace(conjunct, replaceMap)).collect(ImmutableSet.toImmutableSet());
newChildren.add(new LogicalFilter<>(newFilterPredicates, child));
}
-
if (allOneRowRelation) {
- return filter;
- }
-
- if (setOperation instanceof LogicalUnion && setOperation.getQualifier() == Qualifier.DISTINCT) {
- return new LogicalFilter<>(filter.getConjuncts(),
- ((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren));
+ return f;
}
if (hasOneRowRelation) {
// If there are some `OneRowRelation` exists, we need to keep the `filter`.
- return filter.withChildren(
- ((LogicalUnion) setOperation).withHasPushedFilter().withNewChildren(newChildren));
+ return f.withChildren(((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren));
}
- return setOperation.withNewChildren(newChildren);
+ return setOperation.withChildren(newChildren);
}).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
index cd55848405..f11e9782a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
@@ -80,9 +80,4 @@ public class LogicalExcept extends LogicalSetOperation {
public LogicalExcept withNewOutputs(List<NamedExpression> newOutputs) {
return new LogicalExcept(qualifier, newOutputs, Optional.empty(), Optional.empty(), children);
}
-
- @Override
- public LogicalExcept withNewChildren(List<Plan> children) {
- return withChildren(children);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
index 2f3b37453e..6c831da1d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
@@ -82,9 +82,4 @@ public class LogicalIntersect extends LogicalSetOperation {
return new LogicalIntersect(qualifier, newOutputs,
Optional.empty(), Optional.empty(), children);
}
-
- @Override
- public LogicalIntersect withNewChildren(List<Plan> children) {
- return withChildren(children);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java
index 31d8ab6dcf..308cd2a9e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java
@@ -224,7 +224,4 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements
public int getArity() {
return children.size();
}
-
- public abstract LogicalSetOperation withNewChildren(List<Plan> children);
-
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
index 1f53f89d6d..934df533f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
@@ -116,11 +116,6 @@ public class LogicalUnion extends LogicalSetOperation implements OutputPrunable
return new LogicalUnion(qualifier, outputs, true, Optional.empty(), Optional.empty(), children);
}
- @Override
- public LogicalUnion withNewChildren(List<Plan> children) {
- return withChildren(children);
- }
-
@Override
public LogicalUnion pruneOutputs(List<NamedExpression> prunedOutputs) {
return withNewOutputs(prunedOutputs);
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out
index 319e8a5922..4997e1c25c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out
@@ -8,64 +8,62 @@ CteAnchor[cteId= ( CTEId#3=] )
----------hashAgg[GLOBAL]
------------PhysicalDistribute
--------------hashAgg[LOCAL]
-----------------filter(((all_sales.d_year = 1998) OR (all_sales.d_year = 1999)))
-------------------PhysicalUnion
---------------------hashAgg[GLOBAL]
-----------------------PhysicalDistribute
-------------------------hashAgg[LOCAL]
---------------------------filter(((d_year = 1998) OR (d_year = 1999)))
-----------------------------PhysicalUnion
+----------------PhysicalUnion
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalUnion
+--------------------------PhysicalProject
+----------------------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number)
------------------------------PhysicalProject
---------------------------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number)
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[catalog_returns]
-----------------------------------PhysicalProject
-------------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)
---------------------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk)
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[catalog_sales]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
-----------------------------------------------PhysicalOlapScan[item]
---------------------------------------PhysicalDistribute
-----------------------------------------PhysicalProject
-------------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
---------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalOlapScan[catalog_returns]
------------------------------PhysicalProject
---------------------------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number)
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[store_returns]
-----------------------------------PhysicalProject
-------------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk)
---------------------------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk)
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
-----------------------------------------------PhysicalOlapScan[item]
---------------------------------------PhysicalDistribute
-----------------------------------------PhysicalProject
-------------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
---------------------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number)
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[web_returns]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk)
-----------------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk)
+--------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)
+----------------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk)
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[catalog_sales]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
+------------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
+----------------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalProject
+----------------------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number)
------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_sales]
-------------------------------PhysicalDistribute
---------------------------------PhysicalProject
-----------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
-------------------------------------PhysicalOlapScan[item]
+--------------------------------PhysicalOlapScan[store_returns]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk)
+----------------------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk)
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[store_sales]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
+------------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
+----------------------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number)
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[web_returns]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk)
+--------------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk)
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_sales]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
---------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
-----------------------------------PhysicalOlapScan[date_dim]
+--------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home'))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalDistribute
+----------------------------PhysicalProject
+------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999)))
+--------------------------------PhysicalOlapScan[date_dim]
--PhysicalTopN
----PhysicalDistribute
------PhysicalTopN
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org