You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2026/04/15 12:33:55 UTC

(doris) branch fe_local_shuffle updated: Revert "[fix](fe) Fix BUCKET_SHUFFLE data loss in serial exchange with FE-planned local shuffle"

This is an automated email from the ASF dual-hosted git repository.

huajianlan pushed a commit to branch fe_local_shuffle
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/fe_local_shuffle by this push:
     new 5c517f63b94 Revert "[fix](fe) Fix BUCKET_SHUFFLE data loss in serial exchange with FE-planned local shuffle"
5c517f63b94 is described below

commit 5c517f63b946cbcb12e40771a7b69159bf964174
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Apr 15 20:28:28 2026 +0800

    Revert "[fix](fe) Fix BUCKET_SHUFFLE data loss in serial exchange with FE-planned local shuffle"
    
    This reverts commit 782dc35888d6d744a6fba1b5f271e21245515070.
---
 .../trees/plans/distribute/DistributePlanner.java  | 48 +++++-----------------
 1 file changed, 11 insertions(+), 37 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
index cedaff46f57..48a7ba39413 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
@@ -60,13 +60,11 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Consumer;
 
 /** DistributePlanner */
@@ -247,21 +245,12 @@ public class DistributePlanner {
         boolean useLocalShuffle = receiverPlan.getInstanceJobs().stream()
                 .anyMatch(LocalShuffleAssignedJob.class::isInstance);
         if (useLocalShuffle) {
+            // For BUCKET_SHUFFLE with local shuffle, each instance handles specific buckets.
+            // The exchange must route data to each instance individually, not just the first
+            // per worker. Otherwise instances 1..N-1 create ExchangeSource tasks that never
+            // receive data or EOS, causing a hang.
             if (linkNode.getPartitionType() == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED) {
-                // When Exchange is serial on BE (1 task per worker), only instance 0
-                // creates VDataStreamRecvr. Data sent to other instances is silently
-                // dropped (EOF). Route all data to the first instance per worker;
-                // the FE-planned local exchange (BUCKET_HASH_SHUFFLE) redistributes
-                // locally to the correct join build tasks.
-                // When Exchange is non-serial, each instance has its own Exchange task,
-                // so route specific bucket data to each instance directly.
-                PlanFragment receiverFragment = receiverPlan.getFragmentJob().getFragment();
-                boolean exchangeWillBeSerial = (linkNode.isSerialOperator()
-                        || receiverFragment.hasSerialScanNode())
-                        && receiverFragment.useSerialSource(ConnectContext.get());
-                if (!exchangeWillBeSerial) {
-                    return receiverPlan.getInstanceJobs();
-                }
+                return receiverPlan.getInstanceJobs();
             }
             return getFirstInstancePerWorker(receiverPlan.getInstanceJobs());
         } else if (enableShareHashTableForBroadcastJoin && linkNode.isRightChildOfBroadcastHashJoin()) {
@@ -273,20 +262,15 @@ public class DistributePlanner {
 
     private List<AssignedJob> sortDestinationInstancesByBuckets(
             PipelineDistributedPlan plan, List<AssignedJob> unsorted, int bucketNum) {
-        // Detect first-instance-per-worker mode: if each worker appears at most once,
-        // we must use scan-level buckets (all buckets on that worker) instead of
-        // join-level buckets (only this instance's assigned subset).
-        boolean firstPerWorkerOnly = isFirstPerWorkerOnly(unsorted);
-
         AssignedJob[] instances = new AssignedJob[bucketNum];
         for (AssignedJob instanceJob : unsorted) {
-            // When multiple instances per worker (non-serial exchange), use
-            // assignedJoinBucketIndexes to avoid collisions — each instance handles
-            // only its specific join buckets.
-            // When first-per-worker (serial exchange), use scan-level buckets so
-            // all buckets on the worker map to this single receiver instance.
+            // For LocalShuffleBucketJoinAssignedJob, use assignedJoinBucketIndexes which
+            // gives the specific join bucket(s) this instance handles. The scanSource
+            // buckets include ALL buckets on this worker (shared via pooling scan),
+            // which would cause collisions when multiple instances on the same worker
+            // each handle different join buckets.
             Iterable<Integer> bucketIndices;
-            if (instanceJob instanceof LocalShuffleBucketJoinAssignedJob && !firstPerWorkerOnly) {
+            if (instanceJob instanceof LocalShuffleBucketJoinAssignedJob) {
                 bucketIndices = ((LocalShuffleBucketJoinAssignedJob) instanceJob)
                         .getAssignedJoinBucketIndexes();
             } else {
@@ -317,16 +301,6 @@ public class DistributePlanner {
         return Arrays.asList(instances);
     }
 
-    private boolean isFirstPerWorkerOnly(List<AssignedJob> instances) {
-        Set<DistributedPlanWorker> seen = new HashSet<>();
-        for (AssignedJob instance : instances) {
-            if (!seen.add(instance.getAssignedWorker())) {
-                return false;
-            }
-        }
-        return true;
-    }
-
     private List<AssignedJob> getFirstInstancePerWorker(List<AssignedJob> instances) {
         Map<DistributedPlanWorker, AssignedJob> firstInstancePerWorker = Maps.newLinkedHashMap();
         for (AssignedJob instance : instances) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org