You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/03 14:07:23 UTC

(doris) branch branch-4.1 updated: branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new 01120c9e8d0 branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)
01120c9e8d0 is described below

commit 01120c9e8d06515caef3d3b99b65eddf5103ee02
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri Apr 3 22:07:14 2026 +0800

    branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)
    
    Cherry-picked from #60677
    
    Co-authored-by: HappenLee <ha...@selectdb.com>
---
 .../job/UnassignedScanBucketOlapTableJob.java      | 36 ++++++++++++++++++++++
 .../java/org/apache/doris/qe/SessionVariable.java  |  4 +++
 .../doris/qe/LocalShuffleWithBucketJoinTest.java   |  4 +--
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
index 44731efd3c7..4a75e46fded 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
@@ -478,4 +478,40 @@ public class UnassignedScanBucketOlapTableJob extends AbstractUnassignedScanJob
         }
         return workers;
     }
+
+    @Override
+    protected int degreeOfParallelism(int maxParallel, boolean useLocalShuffleToAddParallel) {
+        Preconditions.checkArgument(maxParallel > 0, "maxParallel must be positive");
+        if (!fragment.getDataPartition().isPartitioned()) {
+            return 1;
+        }
+        if (fragment.queryCacheParam != null) {
+            return maxParallel;
+        }
+        if (scanNodes.size() == 1 && scanNodes.get(0) instanceof OlapScanNode) {
+            OlapScanNode olapScanNode = (OlapScanNode) scanNodes.get(0);
+            ConnectContext connectContext = statementContext.getConnectContext();
+            if (connectContext != null && olapScanNode.shouldUseOneInstance(connectContext)) {
+                return 1;
+            }
+        }
+
+        long tabletNum = 0;
+        for (ScanNode scanNode : scanNodes) {
+            if (scanNode instanceof OlapScanNode) {
+                OlapScanNode olapScanNode = (OlapScanNode) scanNode;
+                tabletNum = olapScanNode.getTotalTabletsNum();
+                break;
+            }
+        }
+
+        ConnectContext connectContext = statementContext.getConnectContext();
+        int colocateMaxParallelNum = 128;
+        if (connectContext != null) {
+            colocateMaxParallelNum = connectContext.getSessionVariable().colocateMaxParallelNum;
+        }
+
+        int maxParallelism = (int) Math.max(tabletNum, fragment.getParallelExecNum());
+        return Math.min(maxParallelism, colocateMaxParallelNum);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1fd733e6db9..b5c60ec792e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -170,6 +170,7 @@ public class SessionVariable implements Serializable, Writable {
             "enable_distinct_streaming_agg_force_passthrough";
     public static final String ENABLE_BROADCAST_JOIN_FORCE_PASSTHROUGH = "enable_broadcast_join_force_passthrough";
     public static final String DISABLE_COLOCATE_PLAN = "disable_colocate_plan";
+    public static final String COLOCATE_MAX_PARALLEL_NUM = "colocate_max_parallel_num";
     public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
     public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num";
     public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num";
@@ -1320,6 +1321,9 @@ public class SessionVariable implements Serializable, Writable {
                         setter = "setFragmentInstanceNum", varType = VariableAnnotation.DEPRECATED)
     public int parallelExecInstanceNum = 8;
 
+    @VariableMgr.VarAttr(name = COLOCATE_MAX_PARALLEL_NUM, needForward = true, fuzzy = false)
+    public int colocateMaxParallelNum = 128;
+
     @VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true, needForward = true,
                         setter = "setPipelineTaskNum")
     public int parallelPipelineTaskNum = 0;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
index 0587f8ba61e..ea24aa0c557 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
@@ -67,7 +67,7 @@ public class LocalShuffleWithBucketJoinTest extends TestWithFeService {
         PipelineDistributedPlan distributedPlan
                 = (PipelineDistributedPlan) distributedPlans.get(0);
         List<AssignedJob> instances = distributedPlan.getInstanceJobs();
-        Assertions.assertEquals(beNum * parallelPipelineTaskNum, instances.size());
+        Assertions.assertEquals(beNum * bucketNum, instances.size());
         Assertions.assertTrue(instances.stream().allMatch(LocalShuffleBucketJoinAssignedJob.class::isInstance));
 
         long assignedBucketInstanceNum = instances.stream().map(LocalShuffleBucketJoinAssignedJob.class::cast)
@@ -80,7 +80,7 @@ public class LocalShuffleWithBucketJoinTest extends TestWithFeService {
             workerToInstances.put(instance.getAssignedWorker(), instance);
         }
         for (Collection<AssignedJob> instancePerBe : workerToInstances.asMap().values()) {
-            Assertions.assertEquals(parallelPipelineTaskNum, instancePerBe.size());
+            Assertions.assertEquals(bucketNum, instancePerBe.size());
         }
     }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org