You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ja...@apache.org on 2023/06/09 03:53:51 UTC
[doris] branch master updated: [feature](Nereids): push down alias into union outputs. (#20543)
This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 44e20d9087 [feature](Nereids): push down alias into union outputs. (#20543)
44e20d9087 is described below
commit 44e20d9087980a7beee8b20d2ccdea6c096909df
Author: jakevin <ja...@gmail.com>
AuthorDate: Fri Jun 9 11:53:44 2023 +0800
[feature](Nereids): push down alias into union outputs. (#20543)
---
.../org/apache/doris/nereids/rules/RuleSet.java | 6 +-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../rewrite/logical/PushdownAliasIntoUnionAll.java | 86 ++++++++++++++++++++++
.../rewrite/logical/PushdownAliasThroughJoin.java | 35 +++------
.../nereids_tpcds_shape_sf100_p0/shape/query2.out | 11 ++-
5 files changed, 105 insertions(+), 34 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 84f7479fa6..fb8dce6576 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -72,6 +72,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.logical.MergeGenerates;
import org.apache.doris.nereids.rules.rewrite.logical.MergeLimits;
import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
+import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasIntoUnionAll;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownExpressionsInHashCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggregation;
@@ -126,7 +127,6 @@ public class RuleSet {
new PushdownFilterThroughSetOperation(),
new PushdownFilterThroughWindow(),
new PushdownProjectThroughLimit(),
- new PushdownAliasThroughJoin(),
new EliminateOuterJoin(),
new MergeProjects(),
new MergeFilters(),
@@ -135,7 +135,9 @@ public class RuleSet {
new PushdownFilterThroughCTE(),
new PushdownProjectThroughCTE(),
new PushdownFilterThroughCTEAnchor(),
- new PushdownProjectThroughCTEAnchor());
+ new PushdownProjectThroughCTEAnchor(),
+ new PushdownAliasThroughJoin(),
+ new PushdownAliasIntoUnionAll());
public static final List<Rule> IMPLEMENTATION_RULES = planRuleFactories()
.add(new LogicalCTEProduceToPhysicalCTEProduce())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 2c0e6479c7..7dd712d1e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -138,6 +138,7 @@ public enum RuleType {
PUSHDOWN_FILTER_THROUGH_WINDOW(RuleTypeClass.REWRITE),
PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE),
PUSHDOWN_ALIAS_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSHDOWN_ALIAS_INTO_UNION_ALL(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SORT(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasIntoUnionAll.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasIntoUnionAll.java
new file mode 100644
index 0000000000..65ca513b7b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasIntoUnionAll.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.util.PlanUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Pushdown Alias (inside must be Slot) into UnionAll outputs.
+ * <pre>
+ * Project(c1, c2 as c2t)
+ * |
+ * UnionAll output(c1, c2, c3)
+ * ->
+ * Project(c1, c2t)
+ * |
+ * UnionAll output(c1, c2 as c2t, c3)
+ * </pre>
+ */
+public class PushdownAliasIntoUnionAll extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalProject(logicalUnion())
+ .when(project -> project.child().getQualifier() == Qualifier.ALL)
+ .when(project -> project.getProjects().stream().allMatch(expr ->
+ (expr instanceof Slot) || (expr instanceof Alias && ((Alias) expr).child() instanceof Slot)))
+ .when(project -> project.getProjects().stream().anyMatch(expr -> expr instanceof Alias))
+ .then(project -> {
+ LogicalUnion union = project.child();
+ // aliasMap { Slot -> Alias }
+ Map<Slot, Alias> aliasMap = project.getProjects().stream()
+ .filter(namedExpression -> namedExpression instanceof Alias)
+ .map(namedExpression -> (Alias) namedExpression)
+ .collect(Collectors.toMap(
+ alias -> (Slot) (alias.child()),
+ alias -> alias));
+ Preconditions.checkState(!aliasMap.isEmpty(), "aliasMap should not be empty");
+ List<NamedExpression> newOutput = union.getOutputs().stream()
+ .map(ne -> {
+ Slot outSlot = ne.toSlot();
+ Alias alias = aliasMap.get(outSlot);
+ if (alias == null) {
+ return outSlot;
+ }
+ if (ne instanceof Alias) {
+ return alias.withChildren(ImmutableList.of(((Alias) ne).child()));
+ } else {
+ return alias;
+ }
+ })
+ .collect(Collectors.toList());
+ List<NamedExpression> newProjects = project.getProjects().stream().map(NamedExpression::toSlot)
+ .collect(Collectors.toList());
+ return PlanUtils.projectOrSelf(newProjects, union.withNewOutputs(newOutput));
+ }).toRule(RuleType.PUSHDOWN_ALIAS_INTO_UNION_ALL);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java
index 04811beeca..6496b13b9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java
@@ -27,12 +27,13 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -41,22 +42,12 @@ import java.util.stream.Collectors;
* Pushdown Alias (inside must be Slot) through Join.
*/
public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
- private boolean isAllSlotOrAliasSlot(LogicalProject<? extends Plan> project) {
- return project.getProjects().stream().allMatch(expr -> {
- if (expr instanceof Slot) {
- return true;
- }
- if (expr instanceof Alias) {
- return ((Alias) expr).child() instanceof Slot;
- }
- return false;
- });
- }
-
@Override
public Rule build() {
return logicalProject(logicalJoin())
- .when(this::isAllSlotOrAliasSlot)
+ .when(project -> project.getProjects().stream().allMatch(expr ->
+ (expr instanceof Slot) || (expr instanceof Alias && ((Alias) expr).child() instanceof Slot)))
+ .when(project -> project.getProjects().stream().anyMatch(expr -> expr instanceof Alias))
.then(project -> {
LogicalJoin<? extends Plan, ? extends Plan> join = project.child();
// aliasMap { Slot -> List<Alias<Slot>> }
@@ -71,9 +62,7 @@ public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
}
aliases.add(expr);
});
- if (aliasMap.isEmpty()) {
- return null;
- }
+ Preconditions.checkState(!aliasMap.isEmpty(), "aliasMap should not be empty");
List<NamedExpression> newProjects = project.getProjects().stream().map(NamedExpression::toSlot)
.collect(Collectors.toList());
@@ -122,15 +111,9 @@ public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
private List<NamedExpression> createNewOutput(List<Slot> oldOutput,
Map<Expression, List<NamedExpression>> aliasMap) {
- List<NamedExpression> output = Lists.newArrayList();
- oldOutput.stream().forEach(slot -> {
- List<NamedExpression> alias = aliasMap.get(slot);
- if (alias != null) {
- output.addAll(alias);
- } else {
- output.add(slot);
- }
- });
+ List<NamedExpression> output = oldOutput.stream()
+ .flatMap(slot -> aliasMap.getOrDefault(slot, Collections.singletonList(slot)).stream())
+ .collect(Collectors.toList());
return output;
}
}
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out
index cb297a1129..5b2d1c11c3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out
@@ -7,12 +7,11 @@ CteAnchor[cteId= ( CTEId#4=] )
--------hashAgg[LOCAL]
----------PhysicalProject
------------hashJoin[INNER_JOIN](date_dim.d_date_sk = wscs.sold_date_sk)
---------------PhysicalProject
-----------------PhysicalUnion
-------------------PhysicalProject
---------------------PhysicalOlapScan[web_sales]
-------------------PhysicalProject
---------------------PhysicalOlapScan[catalog_sales]
+--------------PhysicalUnion
+----------------PhysicalProject
+------------------PhysicalOlapScan[web_sales]
+----------------PhysicalProject
+------------------PhysicalOlapScan[catalog_sales]
--------------PhysicalDistribute
----------------PhysicalProject
------------------PhysicalOlapScan[date_dim]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org