You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/12/23 06:54:30 UTC
(doris) 02/04: [opt](nereids) optimize push down project (#58370)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch cherry-pick-nested_column_prune_4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9eb6f7c8d7eb4238fdc73f589fe05b6c3b3a5010
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Nov 26 11:39:03 2025 +0800
[opt](nereids) optimize push down project (#58370)
optimize push down project, this can reduce the scan bytes and shuffle
bytes by prune nested column. #57204 related
the sql:
```sql
select coalecse(struct_element(t1.s, 'city'), 'beijing')
from t1 join t2
on t1.id = t2.id
```
original plan:
```
Project(coalecse(struct_element(t1.s, 'city'), 'beijing'))
|
Join(t1.id=t2.id)
/ \
Project(t1.id, t1.s) Project(t2.id)
| |
Scan(t1) Scan(t2)
```
optimize plan:
```
Project(coalecse(slot#3, 'beijing'))
|
Join(t1.id=t2.id)
/ \
Project(t1.id, struct_element(t1.s, 'city')#3) Project(t2.id)
| |
Scan(t1) Scan(t2)
```
(cherry picked from commit c30c0ff78912a05c2451f6d52c149842b19163ed)
---
.../rules/rewrite/AccessPathPlanCollector.java | 25 ++++++++++++
.../nereids/rules/rewrite/PushDownProject.java | 47 ++++++++++++----------
.../rules/rewrite/PruneNestedColumnTest.java | 21 +++++++---
3 files changed, 65 insertions(+), 28 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
index 514f7bb1e8c..ed253167b38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
@@ -19,7 +19,9 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
@@ -28,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
@@ -53,6 +56,28 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementC
return scanSlotToAccessPaths;
}
+ @Override
+ public Void visitLogicalProject(LogicalProject<? extends Plan> project, StatementContext context) {
+ AccessPathExpressionCollector exprCollector
+ = new AccessPathExpressionCollector(context, allSlotToAccessPaths, false);
+ for (NamedExpression output : project.getProjects()) {
+ // e.g. select struct_element(s, 'city') from (select s from tbl)a;
+ // we will not treat the inner `s` access all path
+ if (output instanceof Slot && allSlotToAccessPaths.containsKey(output.getExprId().asInt())) {
+ continue;
+ } else if (output instanceof Alias && output.child(0) instanceof Slot
+ && allSlotToAccessPaths.containsKey(output.getExprId().asInt())) {
+ Slot innerSlot = (Slot) output.child(0);
+ Collection<CollectAccessPathResult> outerSlotAccessPaths = allSlotToAccessPaths.get(
+ output.getExprId().asInt());
+ allSlotToAccessPaths.putAll(innerSlot.getExprId().asInt(), outerSlotAccessPaths);
+ } else {
+ exprCollector.collect(output);
+ }
+ }
+ return project.child().accept(this, context);
+ }
+
@Override
public Void visitLogicalFilter(LogicalFilter<? extends Plan> filter, StatementContext context) {
boolean bottomFilter = filter.child().arity() == 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
index 8b1fbaac8ae..9fbc9413b29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
@@ -48,7 +48,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.function.Function;
/** push down project if the expression instance of PreferPushDownProject */
@@ -320,13 +319,13 @@ public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot {
private static class PushdownProjectHelper {
private final Plan plan;
private final StatementContext statementContext;
- private final Map<Expression, Pair<Slot, Plan>> exprToChildAndSlot;
+ private final Map<Expression, Expression> oldExprToNewExpr;
private final Multimap<Plan, NamedExpression> childToPushDownProjects;
public PushdownProjectHelper(StatementContext statementContext, Plan plan) {
this.statementContext = statementContext;
this.plan = plan;
- this.exprToChildAndSlot = new LinkedHashMap<>();
+ this.oldExprToNewExpr = new LinkedHashMap<>();
this.childToPushDownProjects = ArrayListMultimap.create();
}
@@ -357,32 +356,36 @@ public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot {
}
public <E extends Expression> Optional<E> pushDownExpression(E expression) {
- if (!(expression instanceof PreferPushDownProject
- || (expression instanceof Alias && expression.child(0) instanceof PreferPushDownProject))) {
+ if (!expression.containsType(PreferPushDownProject.class)) {
return Optional.empty();
}
- Pair<Slot, Plan> existPushdown = exprToChildAndSlot.get(expression);
+ Expression existPushdown = oldExprToNewExpr.get(expression);
if (existPushdown != null) {
- return Optional.of((E) existPushdown.first);
+ return Optional.of((E) existPushdown);
}
- Alias pushDownAlias = null;
- if (expression instanceof Alias) {
- pushDownAlias = (Alias) expression;
- } else {
- pushDownAlias = new Alias(statementContext.getNextExprId(), expression);
- }
-
- Set<Slot> inputSlots = expression.getInputSlots();
- for (Plan child : plan.children()) {
- if (child.getOutputSet().containsAll(inputSlots)) {
- Slot remaimSlot = pushDownAlias.toSlot();
- exprToChildAndSlot.put(expression, Pair.of(remaimSlot, child));
- childToPushDownProjects.put(child, pushDownAlias);
- return Optional.of((E) remaimSlot);
+ Expression newExpression = expression.rewriteDownShortCircuit(e -> {
+ if (e instanceof PreferPushDownProject) {
+ List<Plan> children = plan.children();
+ for (int i = 0; i < children.size(); i++) {
+ Plan child = children.get(i);
+ if (child.getOutputSet().containsAll(e.getInputSlots())) {
+ Alias alias = new Alias(statementContext.getNextExprId(), e);
+ Slot slot = alias.toSlot();
+ childToPushDownProjects.put(child, alias);
+ return slot;
+ }
+ }
}
+ return e;
+ });
+
+ if (newExpression != expression) {
+ oldExprToNewExpr.put(expression, newExpression);
+ return Optional.of((E) newExpression);
+ } else {
+ return Optional.empty();
}
- return Optional.empty();
}
public List<Plan> buildNewChildren() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 22e9c5e949f..3d22e1e6c28 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -72,6 +73,7 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
createTable("create table tbl(\n"
+ " id int,\n"
+ + " value int,\n"
+ " s struct<\n"
+ " city: string,\n"
+ " data: array<map<\n"
@@ -83,6 +85,7 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
createTable("create table tbl2(\n"
+ " id2 int,\n"
+ + " value int,\n"
+ " s2 struct<\n"
+ " city2: string,\n"
+ " data2: array<map<\n"
@@ -376,13 +379,13 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
@Test
public void testUnion() throws Throwable {
- assertColumn("select struct_element(s, 'city') from (select s from tbl union all select null)a",
+ assertColumn("select coalesce(struct_element(s, 'city'), 'abc') from (select s from tbl union all select null)a",
"struct<city:text>",
ImmutableList.of(path("s", "city")),
ImmutableList.of()
);
- assertColumn("select * from (select struct_element(s, 'city') from tbl union all select null)a",
+ assertColumn("select * from (select coalesce(struct_element(s, 'city'), 'abc') from tbl union all select null)a",
"struct<city:text>",
ImmutableList.of(path("s", "city")),
ImmutableList.of()
@@ -407,7 +410,7 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
@Test
public void testPushDownThroughJoin() {
PlanChecker.from(connectContext)
- .analyze("select struct_element(s, 'city') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
+ .analyze("select coalesce(struct_element(s, 'city'), 'abc') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
.rewrite()
.matches(
logicalResultSink(
@@ -426,7 +429,9 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
logicalOneRowRelation()
)
).when(p -> {
- Assertions.assertTrue(p.getProjects().size() == 1 && p.getProjects().get(0) instanceof SlotReference);
+ Assertions.assertTrue(p.getProjects().size() == 1 && p.getProjects().get(0) instanceof Alias
+ && p.getProjects().get(0).child(0) instanceof Coalesce
+ && p.getProjects().get(0).child(0).child(0) instanceof Slot);
return true;
})
)
@@ -479,7 +484,9 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
})
)
).when(p -> {
- Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ Assertions.assertTrue(p.getProjects().size() == 2
+ && (p.getProjects().get(0) instanceof SlotReference
+ || (p.getProjects().get(0) instanceof Alias && p.getProjects().get(0).child(0) instanceof SlotReference)));
return true;
})
)
@@ -509,7 +516,9 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
)
)
).when(p -> {
- Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ Assertions.assertTrue(p.getProjects().size() == 2
+ && (p.getProjects().get(0) instanceof SlotReference
+ || p.getProjects().get(0) instanceof Alias && p.getProjects().get(0).child(0) instanceof SlotReference));
return true;
})
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org