You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2026/04/16 07:38:01 UTC

(doris) branch fe_local_shuffle updated: [fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire

This is an automated email from the ASF dual-hosted git repository.

huajianlan pushed a commit to branch fe_local_shuffle
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/fe_local_shuffle by this push:
     new 04449643620 [fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire
04449643620 is described below

commit 04449643620a23ac6a1e165baa52321b4a969075
Author: 924060929 <la...@selectdb.com>
AuthorDate: Thu Apr 16 15:37:45 2026 +0800

    [fix](local shuffle) Match BE toThrift serial condition in FE enforceRequire
    
    FE's enforceRequire checked `children.get(i).isSerialOperator()` to detect
    serial children, but BE's toThrift sets `is_serial_operator = (isSerialOperator()
    || hasSerialScanNode()) && useSerialSource()`. When a fragment has a pooling scan
    (hasSerialScanNode=true) but the Exchange child is non-merging UNPARTITIONED
    (isSerialOperator=false due to mergeInfo!=null), FE missed the serial condition.
    
    This caused FE to skip LOCAL_EXCHANGE insertion for:
    - HashJoin BROADCAST build side: build pipeline had 1 task, probe had N tasks,
      instance 1+ probes had no hash table → data loss
    - Streaming AGG child: same mismatch pattern
    - Analytic eval child: same pattern
    
    Fix: add `|| fragment.hasSerialScanNode()` to serial child checks in
    HashJoinNode, AggregationNode, and AnalyticEvalNode, matching the BE's
    toThrift condition.
    
    Also restore _deferred_exchangers.empty() early return in BE propagation
    to avoid interfering with BE-native _plan_local_exchange.
---
 .../main/java/org/apache/doris/planner/AggregationNode.java   |  7 +++----
 .../main/java/org/apache/doris/planner/AnalyticEvalNode.java  |  4 ++--
 .../src/main/java/org/apache/doris/planner/HashJoinNode.java  | 11 +++++++++--
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index 13e126031ce..9a9c4f85388 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -298,10 +298,9 @@ public class AggregationNode extends PlanNode {
                     && sessionVariable.enableStreamingAggHashJoinForcePassthrough) {
                 requireChild = LocalExchangeTypeRequire.requirePassthrough();
             } else if (fragment != null && fragment.useSerialSource(connectContext)
-                    && children.get(0).isSerialOperator()) {
+                    && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
                 // BE: _child->is_serial_operator() ? PASSTHROUGH : NOOP
-                // useSerialSource gate needed: ScanNode.isSerialOperator() can be true in
-                // non-pooling mode (scanRanges < pptn) but isn't serial on BE without useSerialSource.
+                // Must match BE's toThrift serial condition which includes hasSerialScanNode().
                 requireChild = LocalExchangeTypeRequire.requirePassthrough();
             } else {
                 requireChild = LocalExchangeTypeRequire.noRequire();
@@ -315,7 +314,7 @@ public class AggregationNode extends PlanNode {
                 } else {
                     // Serialize agg, no group key: BE → _child->is_serial_operator() ? PT : NOOP
                     if (fragment != null && fragment.useSerialSource(connectContext)
-                            && children.get(0).isSerialOperator()) {
+                            && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
                         requireChild = LocalExchangeTypeRequire.requirePassthrough();
                     } else {
                         requireChild = LocalExchangeTypeRequire.noRequire();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
index e93ebd9f017..5195d8ca5a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
@@ -202,9 +202,9 @@ public class AnalyticEvalNode extends PlanNode {
                 outputType = LocalExchangeType.NOOP;
             }
         } else if (fragment.useSerialSource(translatorContext.getConnectContext())
-                && children.get(0).isSerialOperator()) {
+                && (children.get(0).isSerialOperator() || fragment.hasSerialScanNode())) {
             // BE base class: _child->is_serial_operator() ? PASSTHROUGH : NOOP
-            // useSerialSource gate: ScanNode.isSerialOperator() can be true in non-pooling mode
+            // Must match BE's toThrift serial condition which includes hasSerialScanNode().
             requireChild = LocalExchangeTypeRequire.requirePassthrough();
             outputType = LocalExchangeType.PASSTHROUGH;
         } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index daff478885c..ea6b92257c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -312,8 +312,15 @@ public class HashJoinNode extends JoinNodeBase {
             // already includes useSerialSource, so the gate is harmless for Exchange children.
             boolean useSerialSource = fragment != null
                     && fragment.useSerialSource(translatorContext.getConnectContext());
-            boolean probeChildSerial = useSerialSource && children.get(0).isSerialOperator();
-            boolean buildChildSerial = useSerialSource && children.get(1).isSerialOperator();
+            // Must match BE's toThrift serial condition: (isSerialOperator() || hasSerialScanNode())
+            // && useSerialSource(). ExchangeNode.isSerialOperator() only covers merging exchanges
+            // (mergeInfo != null), but non-merging UNPARTITIONED exchanges in pooling-scan
+            // fragments are also serial on BE due to hasSerialScanNode().
+            boolean fragmentHasSerialScan = fragment != null && fragment.hasSerialScanNode();
+            boolean probeChildSerial = useSerialSource
+                    && (children.get(0).isSerialOperator() || fragmentHasSerialScan);
+            boolean buildChildSerial = useSerialSource
+                    && (children.get(1).isSerialOperator() || fragmentHasSerialScan);
             probeSideRequire = probeChildSerial
                     ? LocalExchangeTypeRequire.requirePassthrough()
                     : LocalExchangeTypeRequire.noRequire();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org