You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/03 14:07:23 UTC
(doris) branch branch-4.1 updated: branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 01120c9e8d0 branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)
01120c9e8d0 is described below
commit 01120c9e8d06515caef3d3b99b65eddf5103ee02
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri Apr 3 22:07:14 2026 +0800
branch-4.1: [Exec](performance) change colocate execution parallel num #60677 (#61982)
Cherry-picked from #60677
Co-authored-by: HappenLee <ha...@selectdb.com>
---
.../job/UnassignedScanBucketOlapTableJob.java | 36 ++++++++++++++++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 4 +++
.../doris/qe/LocalShuffleWithBucketJoinTest.java | 4 +--
3 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
index 44731efd3c7..4a75e46fded 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
@@ -478,4 +478,40 @@ public class UnassignedScanBucketOlapTableJob extends AbstractUnassignedScanJob
}
return workers;
}
+
+ @Override
+ protected int degreeOfParallelism(int maxParallel, boolean useLocalShuffleToAddParallel) {
+ Preconditions.checkArgument(maxParallel > 0, "maxParallel must be positive");
+ if (!fragment.getDataPartition().isPartitioned()) {
+ return 1;
+ }
+ if (fragment.queryCacheParam != null) {
+ return maxParallel;
+ }
+ if (scanNodes.size() == 1 && scanNodes.get(0) instanceof OlapScanNode) {
+ OlapScanNode olapScanNode = (OlapScanNode) scanNodes.get(0);
+ ConnectContext connectContext = statementContext.getConnectContext();
+ if (connectContext != null && olapScanNode.shouldUseOneInstance(connectContext)) {
+ return 1;
+ }
+ }
+
+ long tabletNum = 0;
+ for (ScanNode scanNode : scanNodes) {
+ if (scanNode instanceof OlapScanNode) {
+ OlapScanNode olapScanNode = (OlapScanNode) scanNode;
+ tabletNum = olapScanNode.getTotalTabletsNum();
+ break;
+ }
+ }
+
+ ConnectContext connectContext = statementContext.getConnectContext();
+ int colocateMaxParallelNum = 128;
+ if (connectContext != null) {
+ colocateMaxParallelNum = connectContext.getSessionVariable().colocateMaxParallelNum;
+ }
+
+ int maxParallelism = (int) Math.max(tabletNum, fragment.getParallelExecNum());
+ return Math.min(maxParallelism, colocateMaxParallelNum);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1fd733e6db9..b5c60ec792e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -170,6 +170,7 @@ public class SessionVariable implements Serializable, Writable {
"enable_distinct_streaming_agg_force_passthrough";
public static final String ENABLE_BROADCAST_JOIN_FORCE_PASSTHROUGH = "enable_broadcast_join_force_passthrough";
public static final String DISABLE_COLOCATE_PLAN = "disable_colocate_plan";
+ public static final String COLOCATE_MAX_PARALLEL_NUM = "colocate_max_parallel_num";
public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num";
public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num";
@@ -1320,6 +1321,9 @@ public class SessionVariable implements Serializable, Writable {
setter = "setFragmentInstanceNum", varType = VariableAnnotation.DEPRECATED)
public int parallelExecInstanceNum = 8;
+ @VariableMgr.VarAttr(name = COLOCATE_MAX_PARALLEL_NUM, needForward = true, fuzzy = false)
+ public int colocateMaxParallelNum = 128;
+
@VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true, needForward = true,
setter = "setPipelineTaskNum")
public int parallelPipelineTaskNum = 0;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
index 0587f8ba61e..ea24aa0c557 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleWithBucketJoinTest.java
@@ -67,7 +67,7 @@ public class LocalShuffleWithBucketJoinTest extends TestWithFeService {
PipelineDistributedPlan distributedPlan
= (PipelineDistributedPlan) distributedPlans.get(0);
List<AssignedJob> instances = distributedPlan.getInstanceJobs();
- Assertions.assertEquals(beNum * parallelPipelineTaskNum, instances.size());
+ Assertions.assertEquals(beNum * bucketNum, instances.size());
Assertions.assertTrue(instances.stream().allMatch(LocalShuffleBucketJoinAssignedJob.class::isInstance));
long assignedBucketInstanceNum = instances.stream().map(LocalShuffleBucketJoinAssignedJob.class::cast)
@@ -80,7 +80,7 @@ public class LocalShuffleWithBucketJoinTest extends TestWithFeService {
workerToInstances.put(instance.getAssignedWorker(), instance);
}
for (Collection<AssignedJob> instancePerBe : workerToInstances.asMap().values()) {
- Assertions.assertEquals(parallelPipelineTaskNum, instancePerBe.size());
+ Assertions.assertEquals(bucketNum, instancePerBe.size());
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org