You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/15 05:04:28 UTC
(doris) branch branch-4.0 updated: branch-4.0 [fix](nereids)Fix CTE consumer stats derivation when producer stats are not precomputed #61074 (#61863)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 876855e6bcd branch-4.0 [fix](nereids)Fix CTE consumer stats derivation when producer stats are not precomputed #61074 (#61863)
876855e6bcd is described below
commit 876855e6bcdb303f0329534fda9b81b7c25ecc48
Author: minghong <zh...@selectdb.com>
AuthorDate: Wed Apr 15 13:04:20 2026 +0800
branch-4.0 [fix](nereids)Fix CTE consumer stats derivation when producer stats are not precomputed #61074 (#61863)
### What problem does this PR solve?
pick61074
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../org/apache/doris/nereids/StatementContext.java | 23 +++++++++++-----------
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 4 +++-
.../doris/nereids/rules/rewrite/OrExpansion.java | 2 ++
.../nereids/rules/rewrite/RewriteCteChildren.java | 23 ++++++++++++++--------
.../rules/rewrite/SplitMultiDistinctStrategy.java | 4 +---
.../doris/nereids/rules/rewrite/StatsDerive.java | 23 ++++++++++++----------
6 files changed, 46 insertions(+), 33 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 6f4b97b9403..6da14c14bcf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -54,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.TableId;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.RelationUtil;
import org.apache.doris.qe.ConnectContext;
@@ -156,7 +157,7 @@ public class StatementContext implements Closeable {
private final Map<CTEId, Set<LogicalCTEConsumer>> cteIdToConsumers = new HashMap<>();
private final Map<CTEId, Set<Slot>> cteIdToOutputIds = new HashMap<>();
- private final Map<CTEId, Statistics> cteIdToProducerStats = new HashMap<>();
+ private final Map<CTEId, LogicalCTEProducer<? extends Plan>> cteIdToProducer = new HashMap<>();
private final Map<RelationId, Set<Expression>> consumerIdToFilters = new HashMap<>();
// Used to update consumer's stats
@@ -638,7 +639,7 @@ public class StatementContext implements Closeable {
return new CteEnvironmentSnapshot(
copyMapOfSets(cteIdToConsumers),
copyMapOfSets(cteIdToOutputIds),
- new HashMap<>(cteIdToProducerStats),
+ new HashMap<>(cteIdToProducer),
copyMapOfSets(consumerIdToFilters),
copyMapOfLists(cteIdToConsumerGroup),
new HashMap<>(rewrittenCteProducer),
@@ -653,8 +654,8 @@ public class StatementContext implements Closeable {
cteIdToOutputIds.clear();
cteIdToOutputIds.putAll(snapshot.cteIdToOutputIds);
- cteIdToProducerStats.clear();
- cteIdToProducerStats.putAll(snapshot.cteIdToProducerStats);
+ cteIdToProducer.clear();
+ cteIdToProducer.putAll(snapshot.cteIdToProducer);
consumerIdToFilters.clear();
consumerIdToFilters.putAll(snapshot.consumerIdToFilters);
@@ -689,7 +690,7 @@ public class StatementContext implements Closeable {
public static class CteEnvironmentSnapshot {
private final Map<CTEId, Set<LogicalCTEConsumer>> cteIdToConsumers;
private final Map<CTEId, Set<Slot>> cteIdToOutputIds;
- private final Map<CTEId, Statistics> cteIdToProducerStats;
+ private final Map<CTEId, LogicalCTEProducer<? extends Plan>> cteIdToProducer;
private final Map<RelationId, Set<Expression>> consumerIdToFilters;
private final Map<CTEId, List<Pair<Multimap<Slot, Slot>, Group>>> cteIdToConsumerGroup;
private final Map<CTEId, LogicalPlan> rewrittenCteProducer;
@@ -701,14 +702,14 @@ public class StatementContext implements Closeable {
public CteEnvironmentSnapshot(
Map<CTEId, Set<LogicalCTEConsumer>> cteIdToConsumers,
Map<CTEId, Set<Slot>> cteIdToOutputIds,
- Map<CTEId, Statistics> cteIdToProducerStats,
+ Map<CTEId, LogicalCTEProducer<? extends Plan>> cteIdToProducer,
Map<RelationId, Set<Expression>> consumerIdToFilters,
Map<CTEId, List<Pair<Multimap<Slot, Slot>, Group>>> cteIdToConsumerGroup,
Map<CTEId, LogicalPlan> rewrittenCteProducer,
Map<CTEId, LogicalPlan> rewrittenCteConsumer) {
this.cteIdToConsumers = cteIdToConsumers;
this.cteIdToOutputIds = cteIdToOutputIds;
- this.cteIdToProducerStats = cteIdToProducerStats;
+ this.cteIdToProducer = cteIdToProducer;
this.consumerIdToFilters = consumerIdToFilters;
this.cteIdToConsumerGroup = cteIdToConsumerGroup;
this.rewrittenCteProducer = rewrittenCteProducer;
@@ -1111,12 +1112,12 @@ public class StatementContext implements Closeable {
return prepareStage;
}
- public Statistics getProducerStatsByCteId(CTEId id) {
- return cteIdToProducerStats.get(id);
+ public LogicalCTEProducer<? extends Plan> getCteProducerByCteId(CTEId id) {
+ return cteIdToProducer.get(id);
}
- public void setProducerStats(CTEId id, Statistics stats) {
- cteIdToProducerStats.put(id, stats);
+ public void setCteProducer(CTEId id, LogicalCTEProducer<? extends Plan> producer) {
+ cteIdToProducer.put(id, producer);
}
public void setIsInsert(boolean isInsert) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index 4287e8ca7de..c49b3f01c51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -109,7 +109,9 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan));
outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
- cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias));
+ LogicalCTEProducer<Plan> cteProducer = new LogicalCTEProducer<>(cteId, logicalSubQueryAlias);
+ cascadesContext.getStatementContext().setCteProducer(cteId, cteProducer);
+ cteProducerPlans.add(cteProducer);
}
return Pair.of(outerCteCtx, cteProducerPlans);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
index 4cb07fd44fd..2048ea8f50e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
@@ -141,10 +141,12 @@ public class OrExpansion extends DefaultPlanRewriter<OrExpandsionContext> implem
.deepCopy((LogicalPlan) join.left(), new DeepCopierContext());
LogicalCTEProducer<? extends Plan> leftProducer = new LogicalCTEProducer<>(
ctx.statementContext.getNextCTEId(), leftClone);
+ ctx.statementContext.setCteProducer(leftProducer.getCteId(), leftProducer);
LogicalPlan rightClone = LogicalPlanDeepCopier.INSTANCE
.deepCopy((LogicalPlan) join.right(), new DeepCopierContext());
LogicalCTEProducer<? extends Plan> rightProducer = new LogicalCTEProducer<>(
ctx.statementContext.getNextCTEId(), rightClone);
+ ctx.statementContext.setCteProducer(rightProducer.getCteId(), rightProducer);
Map<Slot, Slot> leftCloneToLeft = new HashMap<>();
for (int i = 0; i < leftClone.getOutput().size(); i++) {
leftCloneToLeft.put(leftClone.getOutput().get(i), (join.left()).getOutput().get(i));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java
index 31aeb0be1ba..ac8ddbfeada 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java
@@ -53,9 +53,10 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
- * rewrite CteAnchor consumer side and producer side recursively， all CteAnchor must at top of the plan
+ * rewrite CteAnchor consumer side and producer side recursively， all CteAnchor
+ * must at top of the plan
*/
-@DependsRules({PullUpCteAnchor.class, CTEInline.class})
+@DependsRules({ PullUpCteAnchor.class, CTEInline.class })
public class RewriteCteChildren extends DefaultPlanRewriter<CascadesContext> implements CustomRewriter {
private final List<RewriteJob> jobs;
@@ -127,8 +128,8 @@ public class RewriteCteChildren extends DefaultPlanRewriter<CascadesContext> imp
Set<Slot> producerOutputs = cascadesContext.getStatementContext()
.getCteIdToOutputIds().get(cteProducer.getCteId());
if (producerOutputs.size() < child.getOutput().size()) {
- ImmutableList.Builder<NamedExpression> projectsBuilder
- = ImmutableList.builderWithExpectedSize(producerOutputs.size());
+ ImmutableList.Builder<NamedExpression> projectsBuilder = ImmutableList
+ .builderWithExpectedSize(producerOutputs.size());
for (Slot slot : child.getOutput()) {
if (producerOutputs.contains(slot)) {
projectsBuilder.add(slot);
@@ -147,7 +148,10 @@ public class RewriteCteChildren extends DefaultPlanRewriter<CascadesContext> imp
cascadesContext.addPlanProcesses(rewrittenCtx.getPlanProcesses());
cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteProducer.getCteId(), child);
}
- return cteProducer.withChildren(child);
+ LogicalCTEProducer<? extends Plan> rewrittenProducer = (LogicalCTEProducer<? extends Plan>) cteProducer
+ .withChildren(child);
+ cascadesContext.getStatementContext().setCteProducer(rewrittenProducer.getCteId(), rewrittenProducer);
+ return rewrittenProducer;
}
private LogicalPlan pushPlanUnderAnchor(LogicalPlan plan) {
@@ -160,13 +164,16 @@ public class RewriteCteChildren extends DefaultPlanRewriter<CascadesContext> imp
}
/*
- * An expression can only be pushed down if it has filter expressions on all consumers that reference the slot.
- * For example, let's assume a producer has two consumers, consumer1 and consumer2:
+ * An expression can only be pushed down if it has filter expressions on all
+ * consumers that reference the slot.
+ * For example, let's assume a producer has two consumers, consumer1 and
+ * consumer2:
*
* filter(a > 5 and b < 1) -> consumer1
* filter(a < 8) -> consumer2
*
- * In this case, the only expression that can be pushed down to the producer is filter(a > 5 or a < 8).
+ * In this case, the only expression that can be pushed down to the producer is
+ * filter(a > 5 or a < 8).
*/
private LogicalPlan tryToConstructFilter(CascadesContext cascadesContext, CTEId cteId, LogicalPlan child) {
Set<RelationId> consumerIds = cascadesContext.getCteIdToConsumers().get(cteId).stream()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
index 8fc150c7055..4363e7d33ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext;
-import org.apache.doris.nereids.rules.rewrite.StatsDerive.DeriveContext;
import org.apache.doris.nereids.trees.copier.DeepCopierContext;
import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -61,8 +60,7 @@ public class SplitMultiDistinctStrategy {
LogicalCTEProducer<Plan> producer = new LogicalCTEProducer<>(ctx.statementContext.getNextCTEId(),
cloneAgg.child());
ctx.cteProducerList.add(producer);
- StatsDerive derive = new StatsDerive(false);
- producer.accept(derive, new DeriveContext());
+ ctx.statementContext.setCteProducer(producer.getCteId(), producer);
Map<Slot, Slot> originToProducerSlot = new HashMap<>();
for (int i = 0; i < agg.child().getOutput().size(); ++i) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java
index e3e06c23e6b..cc7240a98bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java
@@ -72,8 +72,8 @@ public class StatsDerive extends PlanVisitor<Statistics, StatsDerive.DeriveConte
private final boolean deepDerive;
/**
- * context
- */
+ * context
+ */
public static class DeriveContext {
StatsCalculator calculator = new StatsCalculator(null);
}
@@ -351,20 +351,28 @@ public class StatsDerive extends PlanVisitor<Statistics, StatsDerive.DeriveConte
@Override
public Statistics visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, DeriveContext context) {
+ // Fallback registration: ensure cteId -> producer mapping is always available
+ // even if some upstream rewrite path misses explicit registration.
+ ConnectContext.get().getStatementContext().setCteProducer(cteProducer.getCteId(), cteProducer);
Statistics prodStats = cteProducer.child().accept(this, context);
StatisticsBuilder builder = new StatisticsBuilder(prodStats);
builder.setWidthInJoinCluster(1);
Statistics stats = builder.build();
cteProducer.setStatistics(stats);
- ConnectContext.get().getStatementContext().setProducerStats(cteProducer.getCteId(), stats);
return stats;
}
@Override
public Statistics visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, DeriveContext context) {
CTEId cteId = cteConsumer.getCteId();
- Statistics prodStats = ConnectContext.get().getStatementContext().getProducerStatsByCteId(cteId);
- Preconditions.checkArgument(prodStats != null, String.format("Stats for CTE: %s not found", cteId));
+ LogicalCTEProducer<? extends Plan> cteProducer = ConnectContext.get().getStatementContext()
+ .getCteProducerByCteId(cteId);
+ Preconditions.checkState(cteProducer != null,
+ String.format("CTE producer for CTE: %s not found", cteId));
+ Statistics prodStats = cteProducer.getStats();
+ if (prodStats == null || deepDerive) {
+ prodStats = cteProducer.accept(this, context);
+ }
Statistics consumerStats = new Statistics(prodStats.getRowCount(), 1, new HashMap<>());
for (Slot slot : cteConsumer.getOutput()) {
Slot prodSlot = cteConsumer.getProducerSlot(slot);
@@ -404,8 +412,3 @@ public class StatsDerive extends PlanVisitor<Statistics, StatsDerive.DeriveConte
return stats;
}
}
-
-
-
-
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org