You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2026/04/16 07:38:01 UTC
(doris) branch fe_local_shuffle updated: [fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch fe_local_shuffle
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/fe_local_shuffle by this push:
new 04449643620 [fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire
04449643620 is described below
commit 04449643620a23ac6a1e165baa52321b4a969075
Author: 924060929 <la...@selectdb.com>
AuthorDate: Thu Apr 16 15:37:45 2026 +0800
[fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire
FE's enforceRequire checked `children.get(i).isSerialOperator()` to detect
serial children, but BE's toThrift sets `is_serial_operator = (isSerialOperator()
|| hasSerialScanNode()) && useSerialSource()`. When a fragment has a pooling scan
(hasSerialScanNode=true) but the Exchange child is non-merging UNPARTITIONED
(isSerialOperator=false due to mergeInfo!=null), FE missed the serial condition.
This caused FE to skip LOCAL_EXCHANGE insertion for:
- HashJoin BROADCAST build side: build pipeline had 1 task, probe had N tasks,
instance 1+ probes had no hash table → data loss
- Streaming AGG child: same mismatch pattern
- Analytic eval child: same pattern
Fix: add `|| fragment.hasSerialScanNode()` to serial child checks in
HashJoinNode, AggregationNode, and AnalyticEvalNode, matching the BE's
toThrift condition.
Also restore _deferred_exchangers.empty() early return in BE propagation
to avoid interfering with BE-native _plan_local_exchange.
---
.../main/java/org/apache/doris/planner/AggregationNode.java | 7 +++----
.../main/java/org/apache/doris/planner/AnalyticEvalNode.java | 4 ++--
.../src/main/java/org/apache/doris/planner/HashJoinNode.java | 11 +++++++++--
3 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index 13e126031ce..9a9c4f85388 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -298,10 +298,9 @@ public class AggregationNode extends PlanNode {
&& sessionVariable.enableStreamingAggHashJoinForcePassthrough) {
requireChild = LocalExchangeTypeRequire.requirePassthrough();
} else if (fragment != null && fragment.useSerialSource(connectContext)
- && children.get(0).isSerialOperator()) {
+ && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
// BE: _child->is_serial_operator() ? PASSTHROUGH : NOOP
- // useSerialSource gate needed: ScanNode.isSerialOperator() can be true in
- // non-pooling mode (scanRanges < pptn) but isn't serial on BE without useSerialSource.
+ // Must match BE's toThrift serial condition which includes hasSerialScanNode().
requireChild = LocalExchangeTypeRequire.requirePassthrough();
} else {
requireChild = LocalExchangeTypeRequire.noRequire();
@@ -315,7 +314,7 @@ public class AggregationNode extends PlanNode {
} else {
// Serialize agg, no group key: BE → _child->is_serial_operator() ? PT : NOOP
if (fragment != null && fragment.useSerialSource(connectContext)
- && children.get(0).isSerialOperator()) {
+ && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
requireChild = LocalExchangeTypeRequire.requirePassthrough();
} else {
requireChild = LocalExchangeTypeRequire.noRequire();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
index e93ebd9f017..5195d8ca5a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
@@ -202,9 +202,9 @@ public class AnalyticEvalNode extends PlanNode {
outputType = LocalExchangeType.NOOP;
}
} else if (fragment.useSerialSource(translatorContext.getConnectContext())
- && children.get(0).isSerialOperator()) {
+ && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
// BE base class: _child->is_serial_operator() ? PASSTHROUGH : NOOP
- // useSerialSource gate: ScanNode.isSerialOperator() can be true in non-pooling mode
+ // Must match BE's toThrift serial condition which includes hasSerialScanNode().
requireChild = LocalExchangeTypeRequire.requirePassthrough();
outputType = LocalExchangeType.PASSTHROUGH;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index daff478885c..ea6b92257c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -312,8 +312,15 @@ public class HashJoinNode extends JoinNodeBase {
// already includes useSerialSource, so the gate is harmless for Exchange children.
boolean useSerialSource = fragment != null
&& fragment.useSerialSource(translatorContext.getConnectContext());
- boolean probeChildSerial = useSerialSource && children.get(0).isSerialOperator();
- boolean buildChildSerial = useSerialSource && children.get(1).isSerialOperator();
+ // Must match BE's toThrift serial condition: (isSerialOperator() || hasSerialScanNode())
+ // && useSerialSource(). ExchangeNode.isSerialOperator() only covers merging exchanges
+ // (mergeInfo != null), but non-merging UNPARTITIONED exchanges in pooling-scan
+ // fragments are also serial on BE due to hasSerialScanNode().
+ boolean fragmentHasSerialScan = fragment != null && fragment.hasSerialScanNode();
+ boolean probeChildSerial = useSerialSource
+ && (children.get(0).isSerialOperator() || fragmentHasSerialScan);
+ boolean buildChildSerial = useSerialSource
+ && (children.get(1).isSerialOperator() || fragmentHasSerialScan);
probeSideRequire = probeChildSerial
? LocalExchangeTypeRequire.requirePassthrough()
: LocalExchangeTypeRequire.noRequire();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org