You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2026/05/13 04:40:51 UTC
(doris) branch fe_local_shuffle_rebase2 created (now 6d44b28c3e5)
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a change to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
at 6d44b28c3e5 [fix](build) Add missing <mutex> include in multi_version.h
This branch includes the following new commits:
new a4b1d5d83de [refactor](local shuffle) FE local exchange planning - squashed for rebase
new 4c69e81e7ee [fix](build) Fix compilation issues after rebasing on apache/master
new b35cd309772 [test](local shuffle) Add regression test for enable_local_exchange_before_agg
new 113ef4d24b6 [fix](local shuffle) Align AggregationNode with enable_local_exchange_before_agg
new ddbfca6a329 [fix](local shuffle) Phase-aware HASH requirement for AggregationNode
new 86579216412 [fix](build) Make multi_version.h compile on both libc++ and libstdc++
new 6d44b28c3e5 [fix](build) Add missing <mutex> include in multi_version.h
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/07: [fix](build) Fix compilation issues after rebasing on apache/master
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4c69e81e7ee5b36a162ad83339c13ac1212db867
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue May 12 17:49:30 2026 +0800
[fix](build) Fix compilation issues after rebasing on apache/master
- multi_version.h: use atomic_load/atomic_store for shared_ptr (LLVM 20 libc++ compat)
- memory.cpp: fix std::max type mismatch (long vs int64_t on macOS)
- bucketed_aggregation_sink_operator.h: fix ExchangeType::NOOP → TLocalPartitionType::NOOP
- BucketedAggregationNode.java: rename isSerialOperator → isSerialOperatorOnBe
- AnalyzeWhereSubqueryTest.java: remove unused imports
- LocalShuffleNodeCoverageTest.java: fix CTEScanNode ctor + AnalyticEval NOOP expectations
---
be/src/common/multi_version.h | 11 +++++++----
be/src/exec/common/memory.cpp | 2 +-
be/src/exec/operator/bucketed_aggregation_sink_operator.h | 2 +-
.../org/apache/doris/planner/BucketedAggregationNode.java | 3 ++-
.../nereids/rules/analysis/AnalyzeWhereSubqueryTest.java | 3 ---
.../apache/doris/planner/LocalShuffleNodeCoverageTest.java | 12 +++++-------
6 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/be/src/common/multi_version.h b/be/src/common/multi_version.h
index 441f8dedae3..aadd2ef3237 100644
--- a/be/src/common/multi_version.h
+++ b/be/src/common/multi_version.h
@@ -20,7 +20,6 @@
#pragma once
-#include <atomic>
#include <memory>
/** Allow to store and read-only usage of an object in several threads,
@@ -50,14 +49,18 @@ public:
explicit MultiVersion(std::unique_ptr<const T>&& value) : current_version(std::move(value)) {}
+ // Use the pre-C++20 atomic free functions for shared_ptr instead of
+ // std::atomic<shared_ptr>, which requires a trivially_copyable specialization
+ // not available in all libc++ versions (e.g., LLVM 20).
+
/// Obtain current version for read-only usage. Returns shared_ptr, that manages lifetime of version.
- Version get() const { return current_version.load(); }
+ Version get() const { return std::atomic_load(¤t_version); }
/// Update an object with new version.
void set(std::unique_ptr<const T>&& value) {
- current_version.store(Version {std::move(value)});
+ std::atomic_store(¤t_version, Version {std::move(value)});
}
private:
- std::atomic<Version> current_version;
+ Version current_version;
};
diff --git a/be/src/exec/common/memory.cpp b/be/src/exec/common/memory.cpp
index 4ca4af037b3..ef5887e1ca7 100644
--- a/be/src/exec/common/memory.cpp
+++ b/be/src/exec/common/memory.cpp
@@ -51,7 +51,7 @@ int MemLimiter::available_scanner_count(int ins_idx) const {
int64_t estimated_block_mem_bytes_value = get_estimated_block_mem_bytes();
DCHECK_GT(estimated_block_mem_bytes_value, 0);
- int64_t max_count = std::max(1L, mem_limit_value / estimated_block_mem_bytes_value);
+ int64_t max_count = std::max(int64_t(1), mem_limit_value / estimated_block_mem_bytes_value);
int64_t avail_count = max_count;
int64_t per_count = avail_count / parallelism;
if (serial_operator) {
diff --git a/be/src/exec/operator/bucketed_aggregation_sink_operator.h b/be/src/exec/operator/bucketed_aggregation_sink_operator.h
index 34b8103155e..03161dd79d4 100644
--- a/be/src/exec/operator/bucketed_aggregation_sink_operator.h
+++ b/be/src/exec/operator/bucketed_aggregation_sink_operator.h
@@ -114,7 +114,7 @@ public:
// No local exchange needed — each instance builds its own hash tables independently.
DataDistribution required_data_distribution(RuntimeState* state) const override {
- return DataDistribution(ExchangeType::NOOP);
+ return DataDistribution(TLocalPartitionType::NOOP);
}
size_t get_reserve_mem_size(RuntimeState* state, bool eos) override;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BucketedAggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BucketedAggregationNode.java
index 96dbd579388..f49c316cf24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/BucketedAggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BucketedAggregationNode.java
@@ -20,6 +20,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TBucketedAggregationNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
@@ -97,7 +98,7 @@ public class BucketedAggregationNode extends PlanNode {
}
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialOperatorOnBe(ConnectContext context) {
// Bucketed agg handles group-by keys only (no without-key in initial version),
// so it's never a serial operator.
return false;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
index f813988dd91..4e51d196113 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
@@ -51,9 +51,6 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import mockit.Mock;
-import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
index 3bfaadee8be..9f3fea43869 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
@@ -158,8 +158,7 @@ public class LocalShuffleNodeCoverageTest {
public void testCteAndRecursiveNodesAndEmptySet() {
PlanTranslatorContext ctx = new PlanTranslatorContext();
- CTEScanNode cteScanNode = new CTEScanNode(new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())),
- ScanContext.EMPTY);
+ CTEScanNode cteScanNode = new CTEScanNode(ScanContext.EMPTY);
Pair<PlanNode, LocalExchangeType> cteOutput = cteScanNode.enforceAndDeriveLocalExchange(
ctx, null, LocalExchangeTypeRequire.requireHash());
Assertions.assertEquals(LocalExchangeType.NOOP, cteOutput.second);
@@ -464,11 +463,10 @@ public class LocalShuffleNodeCoverageTest {
AnalyticEvalNode noPartition = new AnalyticEvalNode(nextPlanNodeId(), noPartitionChild,
Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
- // No partition → isSerialNode()=true → enforceChild skips exchange.
- // Output is still PASSTHROUGH (hardcoded for empty partitions).
+ // No partition → isSerialNode()=true → returns NOOP (serial nodes let framework handle).
Pair<PlanNode, LocalExchangeType> noPartitionOutput = noPartition.enforceAndDeriveLocalExchange(
ctx, null, LocalExchangeTypeRequire.requireHash());
- Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, noPartitionOutput.second);
+ Assertions.assertEquals(LocalExchangeType.NOOP, noPartitionOutput.second);
Assertions.assertSame(noPartitionChild, noPartition.getChild(0));
// Analytic with partition but no orderBy, non-colocated → noRequire/NOOP.
@@ -492,8 +490,8 @@ public class LocalShuffleNodeCoverageTest {
Mockito.when(orderedAnalytic.fragment.useSerialSource(Mockito.any())).thenReturn(true);
Pair<PlanNode, LocalExchangeType> orderedOutput = orderedAnalytic.enforceAndDeriveLocalExchange(
ctx, null, LocalExchangeTypeRequire.noRequire());
- Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, orderedOutput.second);
- assertChildLocalExchangeType(orderedAnalytic, 0, LocalExchangeType.PASSTHROUGH);
+ // Serial AnalyticEval returns NOOP — lets framework serial check handle fan-out
+ Assertions.assertEquals(LocalExchangeType.NOOP, orderedOutput.second);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/07: [fix](local shuffle) Align AggregationNode with enable_local_exchange_before_agg
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 113ef4d24b625f7a95c022c8f1beb2db75520480
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue May 12 19:33:39 2026 +0800
[fix](local shuffle) Align AggregationNode with enable_local_exchange_before_agg
Adapt FE AggregationNode.enforceAndDeriveLocalExchange to match
apache/doris#62438: when enable_local_exchange_before_agg=true (default)
and !needsFinalize, all 3 BE agg operators request HASH_SHUFFLE.
- Extract common !needsFinalize && !enableLeBeforeAgg early return
- StreamingAgg: force_passthrough_after_join takes highest priority
- DistinctStreamingAgg: add missing needsFinalize + emptyGroupKeys → NOOP
- AggSink: non-finalize with enableLeBeforeAgg → requireHash;
finalize preserves hasPartitionExprs heuristic
Reduces FE/BE consistency mismatches from 8 to 4 (3 known NLJ + 1
union_all_followed_by_agg over-insertion).
---
.../org/apache/doris/planner/AggregationNode.java | 91 +++++++++++-----------
1 file changed, 47 insertions(+), 44 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index 562199a2e89..ca364960ead 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -275,65 +275,61 @@ public class AggregationNode extends PlanNode {
ConnectContext connectContext = translatorContext.getConnectContext();
SessionVariable sessionVariable = connectContext.getSessionVariable();
+ // PR #62438: when false, non-finalize agg falls back to BE base class.
+ boolean enableLeBeforeAgg = sessionVariable.enableLocalExchangeBeforeAgg;
+ boolean hasKeys = !aggInfo.getGroupingExprs().isEmpty();
+ // Each branch mirrors the corresponding BE operator's required_data_distribution()
+ // check order 1:1. The helper baseClassRequire() expands BE's base class behavior.
LocalExchangeTypeRequire requireChild;
if (canUseDistinctStreamingAgg(sessionVariable)) {
- // DistinctStreamingAggOperatorX in BE
- if (needsFinalize || (!aggInfo.getGroupingExprs().isEmpty() && !useStreamingPreagg)) {
- if (AddLocalExchange.isColocated(this)) {
- requireChild = LocalExchangeTypeRequire.requireHash();
- } else {
- requireChild = parentRequire.autoRequireHash();
- }
+ // DistinctStreamingAggOperatorX
+ if (needsFinalize && !hasKeys) {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ } else if (!needsFinalize && !enableLeBeforeAgg) {
+ requireChild = baseClassRequire(connectContext);
+ } else if (needsFinalize || (hasKeys && !useStreamingPreagg)) {
+ requireChild = AddLocalExchange.isColocated(this)
+ ? LocalExchangeTypeRequire.requireHash()
+ : parentRequire.autoRequireHash();
} else if (sessionVariable.enableDistinctStreamingAggForcePassthrough) {
requireChild = LocalExchangeTypeRequire.requirePassthrough();
- } else if (children.get(0).isSerialOperatorOnBe(connectContext)) {
- // BE base class: _child->is_serial_operator() ? PASSTHROUGH : NOOP
- requireChild = LocalExchangeTypeRequire.requirePassthrough();
} else {
- requireChild = LocalExchangeTypeRequire.noRequire();
+ requireChild = baseClassRequire(connectContext);
}
} else if (useStreamingPreagg) {
- // StreamingAggOperatorX in BE: base class required_data_distribution():
- // _child->is_serial_operator() ? PASSTHROUGH : NOOP
- // Special: directly above HashJoin probe with force-passthrough → PASSTHROUGH.
+ // StreamingAggOperatorX
if (children.get(0) instanceof HashJoinNode
&& sessionVariable.enableStreamingAggHashJoinForcePassthrough) {
requireChild = LocalExchangeTypeRequire.requirePassthrough();
- } else if (children.get(0).isSerialOperatorOnBe(connectContext)) {
- // BE: _child->is_serial_operator() ? PASSTHROUGH : NOOP
- requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else if (!needsFinalize && !enableLeBeforeAgg) {
+ requireChild = baseClassRequire(connectContext);
+ } else if (!hasKeys) {
+ requireChild = needsFinalize
+ ? LocalExchangeTypeRequire.noRequire()
+ : baseClassRequire(connectContext);
} else {
- requireChild = LocalExchangeTypeRequire.noRequire();
+ requireChild = LocalExchangeTypeRequire.requireHash();
}
} else {
- // AggSinkOperatorX in BE: required_data_distribution() override
- if (aggInfo.getGroupingExprs().isEmpty()) {
- if (needsFinalize) {
- // Finalize agg, no group key: NOOP (single serial instance collects all)
- requireChild = LocalExchangeTypeRequire.noRequire();
- } else {
- // Serialize agg, no group key: BE → _child->is_serial_operator() ? PT : NOOP
- if (children.get(0).isSerialOperatorOnBe(connectContext)) {
- requireChild = LocalExchangeTypeRequire.requirePassthrough();
- } else {
- requireChild = LocalExchangeTypeRequire.noRequire();
- }
- }
- } else if (AddLocalExchange.isColocated(this)) {
- // Colocate: BUCKET_HASH_SHUFFLE
- requireChild = LocalExchangeTypeRequire.requireHash();
+ // AggSinkOperatorX
+ if (!hasKeys) {
+ requireChild = needsFinalize
+ ? LocalExchangeTypeRequire.noRequire()
+ : baseClassRequire(connectContext);
+ } else if (!needsFinalize && !enableLeBeforeAgg) {
+ requireChild = baseClassRequire(connectContext);
+ } else if (!needsFinalize || AddLocalExchange.isColocated(this)) {
+ // BE: non-finalize (PR #62438) or colocate → HASH
+ requireChild = parentRequire.autoRequireHash();
+ } else if (hasPartitionExprs(parentRequire)) {
+ // FE-only heuristic: finalize non-colocate with parent hash requirement
+ // → inherit parent's specific hash type.
+ requireChild = parentRequire.autoRequireHash();
} else {
- // Non-colocate, has group key: BE → GLOBAL_HASH
- // FE uses parentRequire as proxy (no full need_to_local_exchange equivalent)
- if (!needsFinalize
- && sessionVariable.getShuffledAggNodeIds().contains(this.getId().asInt())) {
- requireChild = LocalExchangeTypeRequire.requireHash();
- } else if (hasPartitionExprs(parentRequire)) {
- requireChild = parentRequire.autoRequireHash();
- } else {
- requireChild = LocalExchangeTypeRequire.noRequire();
- }
+ // FE-only heuristic: finalize non-colocate without parent hash → skip
+ // LE (child Exchange already provides hash distribution).
+ requireChild = LocalExchangeTypeRequire.noRequire();
}
}
@@ -343,6 +339,13 @@ public class AggregationNode extends PlanNode {
return Pair.of(this, enforceResult.second);
}
+ /** BE base class required_data_distribution: serial child → PASSTHROUGH, else → NOOP. */
+ private LocalExchangeTypeRequire baseClassRequire(ConnectContext connectContext) {
+ return children.get(0).isSerialOperatorOnBe(connectContext)
+ ? LocalExchangeTypeRequire.requirePassthrough()
+ : LocalExchangeTypeRequire.noRequire();
+ }
+
@Override
protected List<Expr> getSemanticPartitionExprs() {
return aggInfo.getGroupingExprs();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/07: [refactor](local shuffle) FE local exchange planning - squashed for rebase
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a4b1d5d83de9abde92792a118d166086b58ae903
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue May 12 16:29:03 2026 +0800
[refactor](local shuffle) FE local exchange planning - squashed for rebase
---
.idea/vcs.xml | 56 +-
.../exec/exchange/local_exchange_sink_operator.cpp | 73 +-
.../exec/exchange/local_exchange_sink_operator.h | 28 +-
.../exchange/local_exchange_source_operator.cpp | 5 +-
.../exec/exchange/local_exchange_source_operator.h | 38 +-
be/src/exec/exchange/local_exchanger.h | 42 +-
be/src/exec/operator/aggregation_sink_operator.h | 8 +-
be/src/exec/operator/analytic_sink_operator.h | 8 +-
be/src/exec/operator/assert_num_rows_operator.h | 2 +-
.../distinct_streaming_aggregation_operator.h | 10 +-
be/src/exec/operator/exchange_source_operator.h | 8 +-
be/src/exec/operator/hashjoin_build_sink.h | 12 +-
be/src/exec/operator/hashjoin_probe_operator.h | 14 +-
.../operator/nested_loop_join_build_operator.h | 6 +-
.../operator/nested_loop_join_probe_operator.h | 4 +-
be/src/exec/operator/operator.cpp | 4 +-
be/src/exec/operator/operator.h | 6 +-
.../exec/operator/partition_sort_sink_operator.h | 4 +-
.../partitioned_hash_join_probe_operator.cpp | 2 -
.../partitioned_hash_join_probe_operator.h | 14 +-
.../operator/partitioned_hash_join_sink_operator.h | 6 +-
.../exec/operator/rec_cte_anchor_sink_operator.h | 2 +-
be/src/exec/operator/rec_cte_sink_operator.h | 2 +-
be/src/exec/operator/rec_cte_source_operator.h | 2 +-
be/src/exec/operator/scan_operator.h | 4 +-
be/src/exec/operator/set_probe_sink_operator.h | 6 +-
be/src/exec/operator/set_sink_operator.h | 4 +-
be/src/exec/operator/set_source_operator.h | 4 +-
be/src/exec/operator/sort_sink_operator.h | 10 +-
.../exec/operator/streaming_aggregation_operator.h | 7 +-
be/src/exec/operator/table_function_operator.h | 2 +-
be/src/exec/operator/union_sink_operator.h | 4 +-
be/src/exec/operator/union_source_operator.h | 4 +-
be/src/exec/pipeline/dependency.h | 50 +-
be/src/exec/pipeline/pipeline.cpp | 3 +-
be/src/exec/pipeline/pipeline.h | 11 +-
be/src/exec/pipeline/pipeline_fragment_context.cpp | 301 +-
be/src/exec/pipeline/pipeline_fragment_context.h | 30 +-
be/src/runtime/exec_env_init.cpp | 11 +
be/src/runtime/runtime_state.h | 7 +-
be/src/udf/python/python_udf_runtime.cpp | 2 -
be/test/exec/pipeline/local_exchanger_test.cpp | 10 +-
be/test/exec/pipeline/pipeline_test.cpp | 21 +-
fe/.idea/vcs.xml | 5 +
.../doris/common/profile/SummaryProfile.java | 8 +
.../doris/datasource/tvf/source/TVFScanNode.java | 11 +
.../org/apache/doris/nereids/NereidsPlanner.java | 20 +-
.../glue/translator/PhysicalPlanTranslator.java | 67 +-
.../glue/translator/PlanTranslatorContext.java | 26 +
.../nereids/rules/rewrite/AccessPathInfo.java | 12 +
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../trees/plans/distribute/DistributePlanner.java | 3 +-
.../job/UnassignedScanBucketOlapTableJob.java | 2 +-
.../org/apache/doris/planner/AddLocalExchange.java | 159 ++
.../org/apache/doris/planner/AggregationNode.java | 105 +-
.../org/apache/doris/planner/AnalyticEvalNode.java | 72 +-
.../apache/doris/planner/AssertNumRowsNode.java | 18 +-
.../java/org/apache/doris/planner/CTEScanNode.java | 10 +
.../java/org/apache/doris/planner/DataSink.java | 5 +
.../org/apache/doris/planner/DataStreamSink.java | 1 +
.../org/apache/doris/planner/EmptySetNode.java | 10 +
.../org/apache/doris/planner/ExchangeNode.java | 33 +-
.../org/apache/doris/planner/HashJoinNode.java | 75 +
.../apache/doris/planner/LocalExchangeNode.java | 316 +++
.../apache/doris/planner/MaterializationNode.java | 16 +-
.../apache/doris/planner/NestedLoopJoinNode.java | 64 +-
.../org/apache/doris/planner/OlapScanNode.java | 18 +
.../apache/doris/planner/PartitionSortNode.java | 31 +
.../org/apache/doris/planner/PlanFragment.java | 2 +-
.../java/org/apache/doris/planner/PlanNode.java | 181 +-
.../org/apache/doris/planner/RecursiveCteNode.java | 24 +
.../apache/doris/planner/RecursiveCteScanNode.java | 12 +-
.../java/org/apache/doris/planner/RepeatNode.java | 18 +-
.../org/apache/doris/planner/RuntimeFilter.java | 2 +-
.../java/org/apache/doris/planner/ScanNode.java | 16 +-
.../java/org/apache/doris/planner/SelectNode.java | 18 +-
.../org/apache/doris/planner/SetOperationNode.java | 49 +
.../java/org/apache/doris/planner/SortNode.java | 47 +-
.../apache/doris/planner/TableFunctionNode.java | 24 +
.../java/org/apache/doris/planner/UnionNode.java | 2 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 5 +-
.../org/apache/doris/qe/NereidsCoordinator.java | 11 +
.../java/org/apache/doris/qe/SessionVariable.java | 22 +
.../java/org/apache/doris/qe/StmtExecutor.java | 4 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 91 +-
.../rules/analysis/AnalyzeWhereSubqueryTest.java | 10 +
.../planner/LocalShuffleNodeCoverageTest.java | 662 +++++
.../apache/doris/qe/LocalExchangePlannerTest.java | 372 +++
gensrc/thrift/PaloInternalService.thrift | 5 +-
gensrc/thrift/Partitions.thrift | 36 +
gensrc/thrift/PlanNodes.thrift | 22 +-
.../test_multilevel_join_agg_local_shuffle.out | 2865 ++++++++++++++++++++
.../plugins/plugin_profile_plan_tree.groovy | 274 ++
.../local_shuffle/print_known_diff_trees.groovy | 80 +
.../test_local_shuffle_fe_be_consistency.groovy | 940 +++++++
.../test_local_shuffle_rqg_bugs.groovy | 1557 +++++++++++
.../test_old_coordinator_local_shuffle.groovy | 102 +
.../test_multilevel_join_agg_local_shuffle.groovy | 879 ++++++
98 files changed, 9945 insertions(+), 322 deletions(-)
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 0a185d9bdc7..fcaf31b9f91 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,34 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
<project version="4">
- <component name="IssueNavigationConfiguration">
- <option name="links">
- <list>
- <IssueNavigationLink>
- <option name="issueRegexp" value="#(\d+)" />
- <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
- </IssueNavigationLink>
- </list>
- </option>
- </component>
- <component name="VcsDirectoryMappings">
- <mapping directory="$PROJECT_DIR$" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
- </component>
-</project>
+ <component name="IssueNavigationConfiguration">
+ <option name="links">
+ <list>
+ <IssueNavigationLink>
+ <option name="issueRegexp" value="#(\d+)" />
+ <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
+ </IssueNavigationLink>
+ </list>
+ </option>
+ </component>
+ <component name="VcsDirectoryMappings">
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/.github/actions/action-pr-title" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/.github/actions/ccache-action" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/.github/actions/get-workflow-origin" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/.github/actions/label-when-approved-action" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/.github/actions/paths-filter" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/faiss" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/openblas" vcs="Git" />
+ </component>
+</project>
\ No newline at end of file
diff --git a/be/src/exec/exchange/local_exchange_sink_operator.cpp b/be/src/exec/exchange/local_exchange_sink_operator.cpp
index 2f9443208e7..eba93289e65 100644
--- a/be/src/exec/exchange/local_exchange_sink_operator.cpp
+++ b/be/src/exec/exchange/local_exchange_sink_operator.cpp
@@ -37,24 +37,31 @@ std::vector<Dependency*> LocalExchangeSinkLocalState::dependencies() const {
return deps;
}
-Status LocalExchangeSinkOperatorX::init(RuntimeState* state, ExchangeType type,
- const int num_buckets, const bool use_global_hash_shuffle,
+Status LocalExchangeSinkOperatorX::init(RuntimeState* state, TLocalPartitionType::type type,
+ const int num_buckets,
const std::map<int, int>& shuffle_idx_to_instance_idx) {
+ DCHECK(!_planned_by_fe);
_name = "LOCAL_EXCHANGE_SINK_OPERATOR(" + get_exchange_type_name(type) + ")";
_type = type;
- if (_type == ExchangeType::HASH_SHUFFLE) {
- _shuffle_idx_to_instance_idx.clear();
- _use_global_shuffle = use_global_hash_shuffle;
+ if (_type == TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE) {
// For shuffle join, if data distribution has been broken by previous operator, we
// should use a HASH_SHUFFLE local exchanger to shuffle data again. To be mentioned,
// we should use map shuffle idx to instance idx because all instances will be
// distributed to all BEs. Otherwise, we should use shuffle idx directly.
- if (use_global_hash_shuffle) {
- _shuffle_idx_to_instance_idx = shuffle_idx_to_instance_idx;
+ _shuffle_idx_to_instance_idx = shuffle_idx_to_instance_idx;
+ if (state->query_options().__isset.enable_new_shuffle_hash_method &&
+ state->query_options().enable_new_shuffle_hash_method) {
+ _partitioner = std::make_unique<Crc32CHashPartitioner>(_num_partitions);
} else {
- for (int i = 0; i < _num_partitions; i++) {
- _shuffle_idx_to_instance_idx[i] = i;
- }
+ _partitioner = std::make_unique<
+ Crc32HashPartitioner<ShuffleChannelIds>>(
+ _num_partitions);
+ }
+ RETURN_IF_ERROR(_partitioner->init(_texprs));
+ } else if (_type == TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE) {
+ _shuffle_idx_to_instance_idx.clear();
+ for (int i = 0; i < _num_partitions; i++) {
+ _shuffle_idx_to_instance_idx[i] = i;
}
if (state->query_options().__isset.enable_new_shuffle_hash_method &&
state->query_options().enable_new_shuffle_hash_method) {
@@ -64,7 +71,7 @@ Status LocalExchangeSinkOperatorX::init(RuntimeState* state, ExchangeType type,
std::make_unique<Crc32HashPartitioner<ShuffleChannelIds>>(_num_partitions);
}
RETURN_IF_ERROR(_partitioner->init(_texprs));
- } else if (_type == ExchangeType::BUCKET_HASH_SHUFFLE) {
+ } else if (_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) {
DCHECK_GT(num_buckets, 0);
_partitioner = std::make_unique<Crc32HashPartitioner<ShuffleChannelIds>>(num_buckets);
RETURN_IF_ERROR(_partitioner->init(_texprs));
@@ -72,9 +79,36 @@ Status LocalExchangeSinkOperatorX::init(RuntimeState* state, ExchangeType type,
return Status::OK();
}
+Status LocalExchangeSinkOperatorX::init_partitioner(RuntimeState* state) {
+ DCHECK(_planned_by_fe);
+ // Set operator name to include exchange type (base class init(tnode) only sets generic name).
+ _name = "LOCAL_EXCHANGE_SINK_OPERATOR(" + get_exchange_type_name(_type) + ")";
+ if (_type == TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE ||
+ _type == TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE) {
+ if (state->query_options().__isset.enable_new_shuffle_hash_method &&
+ state->query_options().enable_new_shuffle_hash_method) {
+ _partitioner = std::make_unique<Crc32CHashPartitioner>(_num_partitions);
+ } else {
+ _partitioner = std::make_unique<
+ Crc32HashPartitioner<ShuffleChannelIds>>(
+ _num_partitions);
+ }
+ RETURN_IF_ERROR(_partitioner->init(_texprs));
+ } else if (_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) {
+ DCHECK_GT(_num_partitions, 0);
+ _partitioner =
+ std::make_unique<Crc32HashPartitioner<ShuffleChannelIds>>(
+ _num_partitions);
+ RETURN_IF_ERROR(_partitioner->init(_texprs));
+ }
+ return Status::OK();
+}
+
Status LocalExchangeSinkOperatorX::prepare(RuntimeState* state) {
RETURN_IF_ERROR(DataSinkOperatorX<LocalExchangeSinkLocalState>::prepare(state));
- if (_type == ExchangeType::HASH_SHUFFLE || _type == ExchangeType::BUCKET_HASH_SHUFFLE) {
+ if (_type == TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE ||
+ _type == TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE ||
+ _type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) {
RETURN_IF_ERROR(_partitioner->prepare(state, _child->row_desc()));
RETURN_IF_ERROR(_partitioner->open(state));
}
@@ -88,11 +122,6 @@ Status LocalExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
SCOPED_TIMER(_init_timer);
_compute_hash_value_timer = ADD_TIMER(custom_profile(), "ComputeHashValueTime");
_distribute_timer = ADD_TIMER(custom_profile(), "DistributeDataTime");
- if (_parent->cast<LocalExchangeSinkOperatorX>()._type == ExchangeType::HASH_SHUFFLE) {
- custom_profile()->add_info_string(
- "UseGlobalShuffle",
- std::to_string(_parent->cast<LocalExchangeSinkOperatorX>()._use_global_shuffle));
- }
custom_profile()->add_info_string(
"PartitionExprsSize",
std::to_string(_parent->cast<LocalExchangeSinkOperatorX>()._partitioned_exprs_num));
@@ -108,8 +137,7 @@ Status LocalExchangeSinkLocalState::open(RuntimeState* state) {
_exchanger = _shared_state->exchanger.get();
DCHECK(_exchanger != nullptr);
- if (_exchanger->get_type() == ExchangeType::HASH_SHUFFLE ||
- _exchanger->get_type() == ExchangeType::BUCKET_HASH_SHUFFLE) {
+ if (is_shuffled_exchange(_exchanger->get_type())) {
auto& p = _parent->cast<LocalExchangeSinkOperatorX>();
RETURN_IF_ERROR(p._partitioner->clone(state, _partitioner));
}
@@ -132,12 +160,11 @@ Status LocalExchangeSinkLocalState::close(RuntimeState* state, Status exec_statu
std::string LocalExchangeSinkLocalState::debug_string(int indentation_level) const {
fmt::memory_buffer debug_string_buffer;
fmt::format_to(debug_string_buffer,
- "{}, _use_global_shuffle: {}, _channel_id: {}, _num_partitions: {}, "
+ "{}, _channel_id: {}, _num_partitions: {}, "
"_num_senders: {}, _num_sources: {}, "
"_running_sink_operators: {}, _running_source_operators: {}",
- Base::debug_string(indentation_level),
- _parent->cast<LocalExchangeSinkOperatorX>()._use_global_shuffle, _channel_id,
- _exchanger->_num_partitions, _exchanger->_num_senders, _exchanger->_num_sources,
+ Base::debug_string(indentation_level), _channel_id, _exchanger->_num_partitions,
+ _exchanger->_num_senders, _exchanger->_num_sources,
_exchanger->_running_sink_operators, _exchanger->_running_source_operators);
return fmt::to_string(debug_string_buffer);
}
diff --git a/be/src/exec/exchange/local_exchange_sink_operator.h b/be/src/exec/exchange/local_exchange_sink_operator.h
index f1ca1935457..eb698fd9736 100644
--- a/be/src/exec/exchange/local_exchange_sink_operator.h
+++ b/be/src/exec/exchange/local_exchange_sink_operator.h
@@ -79,6 +79,17 @@ public:
_texprs(texprs),
_partitioned_exprs_num(texprs.size()),
_shuffle_idx_to_instance_idx(bucket_seq_to_instance_idx) {}
+
+ LocalExchangeSinkOperatorX(int operator_id, int dest_id, const TPlanNode& tnode,
+ int num_partitions,
+ const std::map<int, int>& shuffle_id_to_instance_idx)
+ : Base(operator_id, tnode, dest_id),
+ _type(tnode.local_exchange_node.partition_type),
+ _num_partitions(num_partitions),
+ _texprs(tnode.local_exchange_node.distribute_expr_lists),
+ _partitioned_exprs_num(tnode.local_exchange_node.distribute_expr_lists.size()),
+ _shuffle_idx_to_instance_idx(shuffle_id_to_instance_idx),
+ _planned_by_fe(true) {}
#ifdef BE_TEST
LocalExchangeSinkOperatorX(const std::vector<TExpr>& texprs,
const std::map<int, int>& bucket_seq_to_instance_idx)
@@ -89,18 +100,19 @@ public:
_shuffle_idx_to_instance_idx(bucket_seq_to_instance_idx) {}
#endif
- Status init(const TPlanNode& tnode, RuntimeState* state) override {
- return Status::InternalError("{} should not init with TPlanNode", Base::_name);
- }
-
Status init(const TDataSink& tsink) override {
return Status::InternalError("{} should not init with TPlanNode", Base::_name);
}
- Status init(RuntimeState* state, ExchangeType type, const int num_buckets,
- const bool use_global_hash_shuffle,
+ Status init(RuntimeState* state, TLocalPartitionType::type type, const int num_buckets,
const std::map<int, int>& shuffle_idx_to_instance_idx) override;
+ // Initialize partitioner for FE-planned local exchange nodes. The FE-planned constructor
+ // already sets _type, _num_partitions, _texprs, and _shuffle_idx_to_instance_idx from the
+ // TPlanNode, but does not create the partitioner. This method creates the partitioner so
+ // that prepare() can call _partitioner->prepare() without null dereference.
+ Status init_partitioner(RuntimeState* state);
+
Status prepare(RuntimeState* state) override;
Status sink(RuntimeState* state, Block* in_block, bool eos) override;
@@ -115,13 +127,13 @@ public:
private:
friend class LocalExchangeSinkLocalState;
friend class ShuffleExchanger;
- ExchangeType _type;
+ TLocalPartitionType::type _type;
const int _num_partitions;
const std::vector<TExpr>& _texprs;
const size_t _partitioned_exprs_num;
std::unique_ptr<PartitionerBase> _partitioner;
std::map<int, int> _shuffle_idx_to_instance_idx;
- bool _use_global_shuffle = false;
+ const bool _planned_by_fe = false;
};
} // namespace doris
diff --git a/be/src/exec/exchange/local_exchange_source_operator.cpp b/be/src/exec/exchange/local_exchange_source_operator.cpp
index ad092656f21..d6bf8ac7e21 100644
--- a/be/src/exec/exchange/local_exchange_source_operator.cpp
+++ b/be/src/exec/exchange/local_exchange_source_operator.cpp
@@ -31,8 +31,7 @@ Status LocalExchangeSourceLocalState::init(RuntimeState* state, LocalStateInfo&
DCHECK(_exchanger != nullptr);
_get_block_failed_counter =
ADD_COUNTER_WITH_LEVEL(custom_profile(), "GetBlockFailedTime", TUnit::UNIT, 1);
- if (_exchanger->get_type() == ExchangeType::HASH_SHUFFLE ||
- _exchanger->get_type() == ExchangeType::BUCKET_HASH_SHUFFLE) {
+ if (is_shuffled_exchange(_exchanger->get_type())) {
_copy_data_timer = ADD_TIMER(custom_profile(), "CopyDataTime");
}
@@ -60,7 +59,7 @@ Status LocalExchangeSourceLocalState::close(RuntimeState* state) {
}
std::vector<Dependency*> LocalExchangeSourceLocalState::dependencies() const {
- if ((_exchanger->get_type() == ExchangeType::PASS_TO_ONE) && _channel_id != 0) {
+ if ((_exchanger->get_type() == TLocalPartitionType::PASS_TO_ONE) && _channel_id != 0) {
// If this is a PASS_TO_ONE exchange and is not the first task, source operators always
// return empty result so no dependencies here.
return {};
diff --git a/be/src/exec/exchange/local_exchange_source_operator.h b/be/src/exec/exchange/local_exchange_source_operator.h
index 58252b24ec2..2f3fc2d5294 100644
--- a/be/src/exec/exchange/local_exchange_source_operator.h
+++ b/be/src/exec/exchange/local_exchange_source_operator.h
@@ -63,20 +63,47 @@ class LocalExchangeSourceOperatorX final : public OperatorX<LocalExchangeSourceL
public:
using Base = OperatorX<LocalExchangeSourceLocalState>;
LocalExchangeSourceOperatorX(ObjectPool* pool, int id) : Base(pool, id, id) {}
+ LocalExchangeSourceOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id,
+ const DescriptorTbl& descs)
+ : Base(pool, tnode, operator_id, descs),
+ _exchange_type(tnode.local_exchange_node.partition_type),
+ _planned_by_fe(true) {}
#ifdef BE_TEST
LocalExchangeSourceOperatorX() = default;
#endif
- Status init(ExchangeType type) override {
+ Status init(TLocalPartitionType::type type) override {
+ DCHECK(!_planned_by_fe);
_op_name = "LOCAL_EXCHANGE_OPERATOR(" + get_exchange_type_name(type) + ")";
_exchange_type = type;
return Status::OK();
}
- Status prepare(RuntimeState* state) override { return Status::OK(); }
+ Status prepare(RuntimeState* state) override {
+ if (_planned_by_fe) {
+ RETURN_IF_ERROR(Base::prepare(state));
+ // Base::prepare() resets _op_name from tnode node_type; restore the type-qualified name.
+ _op_name = "LOCAL_EXCHANGE_OPERATOR(" + get_exchange_type_name(_exchange_type) + ")";
+ return Status::OK();
+ }
+ return Status::OK();
+ }
const RowDescriptor& intermediate_row_desc() const override {
+ if (_planned_by_fe) {
+ return Base::intermediate_row_desc();
+ }
return _child->intermediate_row_desc();
}
- RowDescriptor& row_descriptor() override { return _child->row_descriptor(); }
- const RowDescriptor& row_desc() const override { return _child->row_desc(); }
+ RowDescriptor& row_descriptor() override {
+ if (_planned_by_fe) {
+ return Base::row_descriptor();
+ }
+ return _child->row_descriptor();
+ }
+ const RowDescriptor& row_desc() const override {
+ if (_planned_by_fe) {
+ return Base::row_desc();
+ }
+ return _child->row_desc();
+ }
Status get_block(RuntimeState* state, Block* block, bool* eos) override;
@@ -85,7 +112,8 @@ public:
private:
friend class LocalExchangeSourceLocalState;
- ExchangeType _exchange_type;
+ TLocalPartitionType::type _exchange_type;
+ const bool _planned_by_fe = false;
};
} // namespace doris
diff --git a/be/src/exec/exchange/local_exchanger.h b/be/src/exec/exchange/local_exchanger.h
index b2a0166943f..e0a821a3643 100644
--- a/be/src/exec/exchange/local_exchanger.h
+++ b/be/src/exec/exchange/local_exchanger.h
@@ -142,7 +142,7 @@ public:
SourceInfo&& source_info) = 0;
virtual Status sink(RuntimeState* state, Block* in_block, bool eos, Profile&& profile,
SinkInfo& sink_info) = 0;
- virtual ExchangeType get_type() const = 0;
+ virtual TLocalPartitionType::type get_type() const = 0;
// Called if a local exchanger source operator are closed. Free the unused data block in data_queue.
virtual void close(SourceInfo&& source_info) = 0;
// Called if all local exchanger source operators are closed. We free the memory in
@@ -227,16 +227,19 @@ using BlockWrapperSPtr = std::shared_ptr<ExchangerBase::BlockWrapper>;
template <typename BlockType>
class Exchanger : public ExchangerBase {
public:
- Exchanger(int running_sink_operators, int num_partitions, int free_block_limit)
- : ExchangerBase(running_sink_operators, num_partitions, free_block_limit) {
+ Exchanger(int running_sink_operators, int num_partitions, int free_block_limit,
+ TLocalPartitionType::type type)
+ : ExchangerBase(running_sink_operators, num_partitions, free_block_limit), _type(type) {
_data_queue.resize(num_partitions);
_m.resize(num_partitions);
for (size_t i = 0; i < num_partitions; i++) {
_m[i] = std::make_unique<std::mutex>();
}
}
- Exchanger(int running_sink_operators, int num_sources, int num_partitions, int free_block_limit)
- : ExchangerBase(running_sink_operators, num_sources, num_partitions, free_block_limit) {
+ Exchanger(int running_sink_operators, int num_sources, int num_partitions, int free_block_limit,
+ TLocalPartitionType::type type)
+ : ExchangerBase(running_sink_operators, num_sources, num_partitions, free_block_limit),
+ _type(type) {
_data_queue.resize(num_sources);
_m.resize(num_sources);
for (size_t i = 0; i < num_sources; i++) {
@@ -244,6 +247,7 @@ public:
}
}
~Exchanger() override = default;
+ TLocalPartitionType::type get_type() const override { return _type; }
std::string data_queue_debug_string(int i) override {
return fmt::format("Data Queue {}: [size approx = {}, eos = {}]", i,
_data_queue[i].data_queue.size_approx(), _data_queue[i].eos);
@@ -260,6 +264,7 @@ protected:
bool _dequeue_data(BlockType& block, bool* eos, Block* data_block, int channel_id);
std::vector<BlockQueue<BlockType>> _data_queue;
std::vector<std::unique_ptr<std::mutex>> _m;
+ const TLocalPartitionType::type _type;
};
class LocalExchangeSourceLocalState;
@@ -269,9 +274,9 @@ class ShuffleExchanger : public Exchanger<PartitionedBlock> {
public:
ENABLE_FACTORY_CREATOR(ShuffleExchanger);
ShuffleExchanger(int running_sink_operators, int num_sources, int num_partitions,
- int free_block_limit)
+ int free_block_limit, TLocalPartitionType::type type)
: Exchanger<PartitionedBlock>(running_sink_operators, num_sources, num_partitions,
- free_block_limit) {
+ free_block_limit, type) {
DCHECK_GT(num_partitions, 0);
DCHECK_GT(num_sources, 0);
_partition_rows_histogram.resize(running_sink_operators);
@@ -283,7 +288,6 @@ public:
Status get_block(RuntimeState* state, Block* block, bool* eos, Profile&& profile,
SourceInfo&& source_info) override;
void close(SourceInfo&& source_info) override;
- ExchangeType get_type() const override { return ExchangeType::HASH_SHUFFLE; }
protected:
Status _split_rows(RuntimeState* state, const std::vector<uint32_t>& channel_ids, Block* block,
@@ -299,24 +303,22 @@ class BucketShuffleExchanger final : public ShuffleExchanger {
BucketShuffleExchanger(int running_sink_operators, int num_sources, int num_partitions,
int free_block_limit)
: ShuffleExchanger(running_sink_operators, num_sources, num_partitions,
- free_block_limit) {}
+ free_block_limit, TLocalPartitionType::BUCKET_HASH_SHUFFLE) {}
~BucketShuffleExchanger() override = default;
- ExchangeType get_type() const override { return ExchangeType::BUCKET_HASH_SHUFFLE; }
};
class PassthroughExchanger final : public Exchanger<BlockWrapperSPtr> {
public:
ENABLE_FACTORY_CREATOR(PassthroughExchanger);
PassthroughExchanger(int running_sink_operators, int num_partitions, int free_block_limit)
- : Exchanger<BlockWrapperSPtr>(running_sink_operators, num_partitions,
- free_block_limit) {}
+ : Exchanger<BlockWrapperSPtr>(running_sink_operators, num_partitions, free_block_limit,
+ TLocalPartitionType::PASSTHROUGH) {}
~PassthroughExchanger() override = default;
Status sink(RuntimeState* state, Block* in_block, bool eos, Profile&& profile,
SinkInfo& sink_info) override;
Status get_block(RuntimeState* state, Block* block, bool* eos, Profile&& profile,
SourceInfo&& source_info) override;
- ExchangeType get_type() const override { return ExchangeType::PASSTHROUGH; }
void close(SourceInfo&& source_info) override;
};
@@ -324,29 +326,28 @@ class PassToOneExchanger final : public Exchanger<BlockWrapperSPtr> {
public:
ENABLE_FACTORY_CREATOR(PassToOneExchanger);
PassToOneExchanger(int running_sink_operators, int num_partitions, int free_block_limit)
- : Exchanger<BlockWrapperSPtr>(running_sink_operators, num_partitions,
- free_block_limit) {}
+ : Exchanger<BlockWrapperSPtr>(running_sink_operators, num_partitions, free_block_limit,
+ TLocalPartitionType::PASS_TO_ONE) {}
~PassToOneExchanger() override = default;
Status sink(RuntimeState* state, Block* in_block, bool eos, Profile&& profile,
SinkInfo& sink_info) override;
Status get_block(RuntimeState* state, Block* block, bool* eos, Profile&& profile,
SourceInfo&& source_info) override;
- ExchangeType get_type() const override { return ExchangeType::PASS_TO_ONE; }
void close(SourceInfo&& source_info) override;
};
class BroadcastExchanger final : public Exchanger<BroadcastBlock> {
public:
ENABLE_FACTORY_CREATOR(BroadcastExchanger);
BroadcastExchanger(int running_sink_operators, int num_partitions, int free_block_limit)
- : Exchanger<BroadcastBlock>(running_sink_operators, num_partitions, free_block_limit) {}
+ : Exchanger<BroadcastBlock>(running_sink_operators, num_partitions, free_block_limit,
+ TLocalPartitionType::BROADCAST) {}
~BroadcastExchanger() override = default;
Status sink(RuntimeState* state, Block* in_block, bool eos, Profile&& profile,
SinkInfo& sink_info) override;
Status get_block(RuntimeState* state, Block* block, bool* eos, Profile&& profile,
SourceInfo&& source_info) override;
- ExchangeType get_type() const override { return ExchangeType::BROADCAST; }
void close(SourceInfo&& source_info) override;
};
@@ -357,8 +358,8 @@ public:
ENABLE_FACTORY_CREATOR(AdaptivePassthroughExchanger);
AdaptivePassthroughExchanger(int running_sink_operators, int num_partitions,
int free_block_limit)
- : Exchanger<PartitionedBlock>(running_sink_operators, num_partitions,
- free_block_limit) {
+ : Exchanger<PartitionedBlock>(running_sink_operators, num_partitions, free_block_limit,
+ TLocalPartitionType::ADAPTIVE_PASSTHROUGH) {
_partition_rows_histogram.resize(running_sink_operators);
_tmp_eos.resize(num_partitions);
_tmp_block.resize(num_partitions);
@@ -368,7 +369,6 @@ public:
Status get_block(RuntimeState* state, Block* block, bool* eos, Profile&& profile,
SourceInfo&& source_info) override;
- ExchangeType get_type() const override { return ExchangeType::ADAPTIVE_PASSTHROUGH; }
void close(SourceInfo&& source_info) override;
diff --git a/be/src/exec/operator/aggregation_sink_operator.h b/be/src/exec/operator/aggregation_sink_operator.h
index f0cc6b70d4e..ea7c19bcb82 100644
--- a/be/src/exec/operator/aggregation_sink_operator.h
+++ b/be/src/exec/operator/aggregation_sink_operator.h
@@ -159,7 +159,7 @@ public:
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_partition_exprs.empty()) {
return _needs_finalize
- ? DataDistribution(ExchangeType::NOOP)
+ ? DataDistribution(TLocalPartitionType::NOOP)
: DataSinkOperatorX<AggSinkLocalState>::required_data_distribution(
state);
}
@@ -167,8 +167,10 @@ public:
return DataSinkOperatorX<AggSinkLocalState>::required_data_distribution(state);
}
return _is_colocate && _require_bucket_distribution
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
bool is_colocated_operator() const override { return _is_colocate; }
bool is_shuffled_operator() const override {
diff --git a/be/src/exec/operator/analytic_sink_operator.h b/be/src/exec/operator/analytic_sink_operator.h
index 9a64ba1d0b7..2a989020a7f 100644
--- a/be/src/exec/operator/analytic_sink_operator.h
+++ b/be/src/exec/operator/analytic_sink_operator.h
@@ -212,11 +212,13 @@ public:
Status sink(RuntimeState* state, Block* in_block, bool eos) override;
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_partition_by_eq_expr_ctxs.empty()) {
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
} else {
return _is_colocate && _require_bucket_distribution
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
}
diff --git a/be/src/exec/operator/assert_num_rows_operator.h b/be/src/exec/operator/assert_num_rows_operator.h
index 29da76e8a96..4c71266c626 100644
--- a/be/src/exec/operator/assert_num_rows_operator.h
+++ b/be/src/exec/operator/assert_num_rows_operator.h
@@ -47,7 +47,7 @@ public:
[[nodiscard]] bool is_source() const override { return false; }
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
}
private:
diff --git a/be/src/exec/operator/distinct_streaming_aggregation_operator.h b/be/src/exec/operator/distinct_streaming_aggregation_operator.h
index abf42eb50cc..cb4156c9864 100644
--- a/be/src/exec/operator/distinct_streaming_aggregation_operator.h
+++ b/be/src/exec/operator/distinct_streaming_aggregation_operator.h
@@ -116,7 +116,7 @@ public:
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_needs_finalize && _probe_expr_ctxs.empty()) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
if (!_needs_finalize && !state->enable_local_exchange_before_agg()) {
return StatefulOperatorX<DistinctStreamingAggLocalState>::required_data_distribution(
@@ -124,11 +124,13 @@ public:
}
if (_needs_finalize || (!_probe_expr_ctxs.empty() && !_is_streaming_preagg)) {
return _is_colocate && _require_bucket_distribution
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
if (state->enable_distinct_streaming_agg_force_passthrough()) {
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
} else {
return StatefulOperatorX<DistinctStreamingAggLocalState>::required_data_distribution(
state);
diff --git a/be/src/exec/operator/exchange_source_operator.h b/be/src/exec/operator/exchange_source_operator.h
index da00e088586..d47250080a1 100644
--- a/be/src/exec/operator/exchange_source_operator.h
+++ b/be/src/exec/operator/exchange_source_operator.h
@@ -112,13 +112,13 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (OperatorX<ExchangeLocalState>::is_serial_operator()) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
return _partition_type == TPartitionType::HASH_PARTITIONED
- ? DataDistribution(ExchangeType::HASH_SHUFFLE)
+ ? DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE)
: _partition_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE)
- : DataDistribution(ExchangeType::NOOP);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE)
+ : DataDistribution(TLocalPartitionType::NOOP);
}
private:
diff --git a/be/src/exec/operator/hashjoin_build_sink.h b/be/src/exec/operator/hashjoin_build_sink.h
index be77ef6cc69..cf546e6dddb 100644
--- a/be/src/exec/operator/hashjoin_build_sink.h
+++ b/be/src/exec/operator/hashjoin_build_sink.h
@@ -134,15 +134,17 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
} else if (_is_broadcast_join) {
- return _child->is_serial_operator() ? DataDistribution(ExchangeType::PASS_TO_ONE)
- : DataDistribution(ExchangeType::NOOP);
+ return _child->is_serial_operator() ? DataDistribution(TLocalPartitionType::PASS_TO_ONE)
+ : DataDistribution(TLocalPartitionType::NOOP);
}
return _join_distribution == TJoinDistributionType::BUCKET_SHUFFLE ||
_join_distribution == TJoinDistributionType::COLOCATE
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
bool is_shuffled_operator() const override {
diff --git a/be/src/exec/operator/hashjoin_probe_operator.h b/be/src/exec/operator/hashjoin_probe_operator.h
index da60e8b5231..371a43fe1eb 100644
--- a/be/src/exec/operator/hashjoin_probe_operator.h
+++ b/be/src/exec/operator/hashjoin_probe_operator.h
@@ -135,21 +135,23 @@ public:
bool need_more_input_data(RuntimeState* state) const override;
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
} else if (_is_broadcast_join) {
if (state->enable_broadcast_join_force_passthrough()) {
- return DataDistribution(ExchangeType::PASSTHROUGH);
+ return DataDistribution(TLocalPartitionType::PASSTHROUGH);
} else {
return _child && _child->is_serial_operator()
- ? DataDistribution(ExchangeType::PASSTHROUGH)
- : DataDistribution(ExchangeType::NOOP);
+ ? DataDistribution(TLocalPartitionType::PASSTHROUGH)
+ : DataDistribution(TLocalPartitionType::NOOP);
}
}
return (_join_distribution == TJoinDistributionType::BUCKET_SHUFFLE ||
_join_distribution == TJoinDistributionType::COLOCATE
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs));
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs));
}
bool is_broadcast_join() const { return _is_broadcast_join; }
diff --git a/be/src/exec/operator/nested_loop_join_build_operator.h b/be/src/exec/operator/nested_loop_join_build_operator.h
index 265a8767b3e..b6be8eb9254 100644
--- a/be/src/exec/operator/nested_loop_join_build_operator.h
+++ b/be/src/exec/operator/nested_loop_join_build_operator.h
@@ -68,10 +68,10 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
- return _child->is_serial_operator() ? DataDistribution(ExchangeType::BROADCAST)
- : DataDistribution(ExchangeType::NOOP);
+ return _child->is_serial_operator() ? DataDistribution(TLocalPartitionType::BROADCAST)
+ : DataDistribution(TLocalPartitionType::NOOP);
}
private:
diff --git a/be/src/exec/operator/nested_loop_join_probe_operator.h b/be/src/exec/operator/nested_loop_join_probe_operator.h
index aea6972f428..40fecd8f4ec 100644
--- a/be/src/exec/operator/nested_loop_join_probe_operator.h
+++ b/be/src/exec/operator/nested_loop_join_probe_operator.h
@@ -239,9 +239,9 @@ public:
if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
_join_op == TJoinOp::RIGHT_OUTER_JOIN || _join_op == TJoinOp::RIGHT_ANTI_JOIN ||
_join_op == TJoinOp::RIGHT_SEMI_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
- return {ExchangeType::ADAPTIVE_PASSTHROUGH};
+ return {TLocalPartitionType::ADAPTIVE_PASSTHROUGH};
}
const RowDescriptor& row_desc() const override {
diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp
index 96f7933d5d5..491e35ecd80 100644
--- a/be/src/exec/operator/operator.cpp
+++ b/be/src/exec/operator/operator.cpp
@@ -143,8 +143,8 @@ Status PipelineXSinkLocalState<SharedStateArg>::terminate(RuntimeState* state) {
DataDistribution OperatorBase::required_data_distribution(RuntimeState* /*state*/) const {
return _child && _child->is_serial_operator() && !is_source()
- ? DataDistribution(ExchangeType::PASSTHROUGH)
- : DataDistribution(ExchangeType::NOOP);
+ ? DataDistribution(TLocalPartitionType::PASSTHROUGH)
+ : DataDistribution(TLocalPartitionType::NOOP);
}
const RowDescriptor& OperatorBase::row_desc() const {
diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h
index d4b09a4aed3..d1d4cb58e63 100644
--- a/be/src/exec/operator/operator.h
+++ b/be/src/exec/operator/operator.h
@@ -600,8 +600,8 @@ public:
virtual bool reset_to_rerun(RuntimeState* state, OperatorXBase* root) const { return false; }
Status init(const TDataSink& tsink) override;
- [[nodiscard]] virtual Status init(RuntimeState* state, ExchangeType type, const int num_buckets,
- const bool use_global_hash_shuffle,
+ [[nodiscard]] virtual Status init(RuntimeState* state, TLocalPartitionType::type type,
+ const int num_buckets,
const std::map<int, int>& shuffle_idx_to_instance_idx) {
return Status::InternalError("init() is only implemented in local exchange!");
}
@@ -849,7 +849,7 @@ public:
Status init(const TDataSink& tsink) override {
throw Exception(Status::FatalError("should not reach here!"));
}
- virtual Status init(ExchangeType type) {
+ virtual Status init(TLocalPartitionType::type type) {
throw Exception(Status::FatalError("should not reach here!"));
}
[[noreturn]] virtual const std::vector<TRuntimeFilterDesc>& runtime_filter_descs() {
diff --git a/be/src/exec/operator/partition_sort_sink_operator.h b/be/src/exec/operator/partition_sort_sink_operator.h
index 48b709d3f1f..6d098a1c9bd 100644
--- a/be/src/exec/operator/partition_sort_sink_operator.h
+++ b/be/src/exec/operator/partition_sort_sink_operator.h
@@ -95,9 +95,9 @@ public:
Status sink(RuntimeState* state, Block* in_block, bool eos) override;
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_topn_phase == TPartTopNPhase::TWO_PHASE_GLOBAL) {
- return DataDistribution(ExchangeType::HASH_SHUFFLE, _distribute_exprs);
+ return DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE, _distribute_exprs);
}
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
}
size_t get_reserve_mem_size(RuntimeState* state, bool eos) override;
diff --git a/be/src/exec/operator/partitioned_hash_join_probe_operator.cpp b/be/src/exec/operator/partitioned_hash_join_probe_operator.cpp
index 56e80cee3f8..d5c760dbc7b 100644
--- a/be/src/exec/operator/partitioned_hash_join_probe_operator.cpp
+++ b/be/src/exec/operator/partitioned_hash_join_probe_operator.cpp
@@ -523,8 +523,6 @@ PartitionedHashJoinProbeOperatorX::PartitionedHashJoinProbeOperatorX(ObjectPool*
int operator_id,
const DescriptorTbl& descs)
: JoinProbeOperatorX<PartitionedHashJoinProbeLocalState>(pool, tnode, operator_id, descs),
- _join_distribution(tnode.hash_join_node.__isset.dist_type ? tnode.hash_join_node.dist_type
- : TJoinDistributionType::NONE),
_distribution_partition_exprs(tnode.__isset.distribute_expr_lists
? tnode.distribute_expr_lists[0]
: std::vector<TExpr> {}),
diff --git a/be/src/exec/operator/partitioned_hash_join_probe_operator.h b/be/src/exec/operator/partitioned_hash_join_probe_operator.h
index 15767504d85..d9bca3aab30 100644
--- a/be/src/exec/operator/partitioned_hash_join_probe_operator.h
+++ b/be/src/exec/operator/partitioned_hash_join_probe_operator.h
@@ -228,16 +228,8 @@ public:
Status pull(doris::RuntimeState* state, Block* output_block, bool* eos) const override;
bool need_more_input_data(RuntimeState* state) const override;
- DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
- return {ExchangeType::NOOP};
- }
- return (_join_distribution == TJoinDistributionType::BUCKET_SHUFFLE ||
- _join_distribution == TJoinDistributionType::COLOCATE
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE,
- _distribution_partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE,
- _distribution_partition_exprs));
+ DataDistribution required_data_distribution(RuntimeState* state) const override {
+ return _inner_probe_operator->required_data_distribution(state);
}
size_t revocable_mem_size(RuntimeState* state) const override;
@@ -285,8 +277,6 @@ private:
RuntimeState* state, Block* output_block,
bool* eos) const;
- const TJoinDistributionType::type _join_distribution;
-
std::shared_ptr<HashJoinBuildSinkOperatorX> _inner_sink_operator;
std::shared_ptr<HashJoinProbeOperatorX> _inner_probe_operator;
diff --git a/be/src/exec/operator/partitioned_hash_join_sink_operator.h b/be/src/exec/operator/partitioned_hash_join_sink_operator.h
index c6dc2cdfb94..e3c818a33c8 100644
--- a/be/src/exec/operator/partitioned_hash_join_sink_operator.h
+++ b/be/src/exec/operator/partitioned_hash_join_sink_operator.h
@@ -127,14 +127,14 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
return _join_distribution == TJoinDistributionType::BUCKET_SHUFFLE ||
_join_distribution == TJoinDistributionType::COLOCATE
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE,
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
_distribution_partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE,
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
_distribution_partition_exprs);
}
diff --git a/be/src/exec/operator/rec_cte_anchor_sink_operator.h b/be/src/exec/operator/rec_cte_anchor_sink_operator.h
index 42661ee4181..e49ccf7b8c1 100644
--- a/be/src/exec/operator/rec_cte_anchor_sink_operator.h
+++ b/be/src/exec/operator/rec_cte_anchor_sink_operator.h
@@ -66,7 +66,7 @@ public:
bool is_serial_operator() const override { return true; }
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
Status terminate(RuntimeState* state) override {
diff --git a/be/src/exec/operator/rec_cte_sink_operator.h b/be/src/exec/operator/rec_cte_sink_operator.h
index a071cf55761..d42a6da4e35 100644
--- a/be/src/exec/operator/rec_cte_sink_operator.h
+++ b/be/src/exec/operator/rec_cte_sink_operator.h
@@ -77,7 +77,7 @@ public:
bool is_serial_operator() const override { return true; }
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
Status sink(RuntimeState* state, Block* input_block, bool eos) override {
diff --git a/be/src/exec/operator/rec_cte_source_operator.h b/be/src/exec/operator/rec_cte_source_operator.h
index 83e55937998..18ec1a0f267 100644
--- a/be/src/exec/operator/rec_cte_source_operator.h
+++ b/be/src/exec/operator/rec_cte_source_operator.h
@@ -206,7 +206,7 @@ public:
bool is_serial_operator() const override { return true; }
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
Status get_block(RuntimeState* state, Block* block, bool* eos) override {
diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h
index 591bdcac5ee..e816b00b0cc 100644
--- a/be/src/exec/operator/scan_operator.h
+++ b/be/src/exec/operator/scan_operator.h
@@ -366,9 +366,9 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (OperatorX<LocalStateType>::is_serial_operator()) {
// `is_serial_operator()` returns true means we ignore the distribution.
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
- return {ExchangeType::BUCKET_HASH_SHUFFLE};
+ return {TLocalPartitionType::BUCKET_HASH_SHUFFLE};
}
void set_low_memory_mode(RuntimeState* state) override {
diff --git a/be/src/exec/operator/set_probe_sink_operator.h b/be/src/exec/operator/set_probe_sink_operator.h
index c736e2af229..ff47755d3d0 100644
--- a/be/src/exec/operator/set_probe_sink_operator.h
+++ b/be/src/exec/operator/set_probe_sink_operator.h
@@ -103,8 +103,10 @@ public:
Status sink(RuntimeState* state, Block* in_block, bool eos) override;
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return _is_colocate ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ return _is_colocate ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
std::shared_ptr<BasicSharedState> create_shared_state() const override { return nullptr; }
diff --git a/be/src/exec/operator/set_sink_operator.h b/be/src/exec/operator/set_sink_operator.h
index e1c937f1471..154fa7969d6 100644
--- a/be/src/exec/operator/set_sink_operator.h
+++ b/be/src/exec/operator/set_sink_operator.h
@@ -114,8 +114,8 @@ public:
Status sink(RuntimeState* state, Block* in_block, bool eos) override;
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return _is_colocate ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ return _is_colocate ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE, _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE, _partition_exprs);
}
size_t get_reserve_mem_size(RuntimeState* state, bool eos) override;
diff --git a/be/src/exec/operator/set_source_operator.h b/be/src/exec/operator/set_source_operator.h
index 40887f35fab..3655cc86bbf 100644
--- a/be/src/exec/operator/set_source_operator.h
+++ b/be/src/exec/operator/set_source_operator.h
@@ -81,8 +81,8 @@ public:
bool is_shuffled_operator() const override { return true; }
bool is_colocated_operator() const override { return _is_colocate; }
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return _is_colocate ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE)
- : DataDistribution(ExchangeType::HASH_SHUFFLE);
+ return _is_colocate ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
}
Status get_block(RuntimeState* state, Block* block, bool* eos) override;
diff --git a/be/src/exec/operator/sort_sink_operator.h b/be/src/exec/operator/sort_sink_operator.h
index eda5bb31b53..62916ca7ca2 100644
--- a/be/src/exec/operator/sort_sink_operator.h
+++ b/be/src/exec/operator/sort_sink_operator.h
@@ -80,13 +80,15 @@ public:
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
if (_is_analytic_sort) {
return _is_colocate && _require_bucket_distribution
- ? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
- : DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ ? DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE,
+ _partition_exprs)
+ : DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
} else if (_merge_by_exchange) {
// The current sort node is used for the ORDER BY
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
} else {
- return {ExchangeType::NOOP};
+ return {TLocalPartitionType::NOOP};
}
}
bool is_colocated_operator() const override { return _is_colocate; }
diff --git a/be/src/exec/operator/streaming_aggregation_operator.h b/be/src/exec/operator/streaming_aggregation_operator.h
index 40a8de28244..80b64de1c9f 100644
--- a/be/src/exec/operator/streaming_aggregation_operator.h
+++ b/be/src/exec/operator/streaming_aggregation_operator.h
@@ -224,18 +224,19 @@ public:
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_child && _child->is_hash_join_probe() &&
state->enable_streaming_agg_hash_join_force_passthrough()) {
- return DataDistribution(ExchangeType::PASSTHROUGH);
+ return DataDistribution(TLocalPartitionType::PASSTHROUGH);
}
if (!_needs_finalize && !state->enable_local_exchange_before_agg()) {
return StatefulOperatorX<StreamingAggLocalState>::required_data_distribution(state);
}
if (_partition_exprs.empty()) {
return _needs_finalize
- ? DataDistribution(ExchangeType::NOOP)
+ ? DataDistribution(TLocalPartitionType::NOOP)
: StatefulOperatorX<StreamingAggLocalState>::required_data_distribution(
state);
}
- return DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);
+ return DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE,
+ _partition_exprs);
}
private:
diff --git a/be/src/exec/operator/table_function_operator.h b/be/src/exec/operator/table_function_operator.h
index 0a55c090596..de6e546af1d 100644
--- a/be/src/exec/operator/table_function_operator.h
+++ b/be/src/exec/operator/table_function_operator.h
@@ -116,7 +116,7 @@ public:
}
DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
- return {ExchangeType::PASSTHROUGH};
+ return {TLocalPartitionType::PASSTHROUGH};
}
Status push(RuntimeState* state, Block* input_block, bool eos) const override {
diff --git a/be/src/exec/operator/union_sink_operator.h b/be/src/exec/operator/union_sink_operator.h
index 4842ab6b243..f172e120ac8 100644
--- a/be/src/exec/operator/union_sink_operator.h
+++ b/be/src/exec/operator/union_sink_operator.h
@@ -116,10 +116,10 @@ public:
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_require_bucket_distribution) {
- return DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _distribute_exprs);
+ return DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE, _distribute_exprs);
}
if (_followed_by_shuffled_operator) {
- return DataDistribution(ExchangeType::HASH_SHUFFLE, _distribute_exprs);
+ return DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE, _distribute_exprs);
}
return Base::required_data_distribution(state);
}
diff --git a/be/src/exec/operator/union_source_operator.h b/be/src/exec/operator/union_source_operator.h
index 748c9a48e60..2b1d3e10d95 100644
--- a/be/src/exec/operator/union_source_operator.h
+++ b/be/src/exec/operator/union_source_operator.h
@@ -101,10 +101,10 @@ public:
DataDistribution required_data_distribution(RuntimeState* state) const override {
if (_require_bucket_distribution) {
- return DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE);
+ return DataDistribution(TLocalPartitionType::BUCKET_HASH_SHUFFLE);
}
if (_followed_by_shuffled_operator) {
- return DataDistribution(ExchangeType::HASH_SHUFFLE);
+ return DataDistribution(TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
}
return Base::required_data_distribution(state);
}
diff --git a/be/src/exec/pipeline/dependency.h b/be/src/exec/pipeline/dependency.h
index dbb4b938ad4..cc36a977e5d 100644
--- a/be/src/exec/pipeline/dependency.h
+++ b/be/src/exec/pipeline/dependency.h
@@ -828,50 +828,44 @@ public:
Status hash_table_init();
};
-enum class ExchangeType : uint8_t {
- NOOP = 0,
- // Shuffle data by Crc32CHashPartitioner
- HASH_SHUFFLE = 1,
- // Round-robin passthrough data blocks.
- PASSTHROUGH = 2,
- // Shuffle data by Crc32HashPartitioner<ShuffleChannelIds> (e.g. same as storage engine).
- BUCKET_HASH_SHUFFLE = 3,
- // Passthrough data blocks to all channels.
- BROADCAST = 4,
- // Passthrough data to channels evenly in an adaptive way.
- ADAPTIVE_PASSTHROUGH = 5,
- // Send all data to the first channel.
- PASS_TO_ONE = 6,
-};
+inline bool is_shuffled_exchange(TLocalPartitionType::type idx) {
+ return idx == TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE ||
+ idx == TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE ||
+ idx == TLocalPartitionType::BUCKET_HASH_SHUFFLE;
+}
-inline std::string get_exchange_type_name(ExchangeType idx) {
+inline std::string get_exchange_type_name(TLocalPartitionType::type idx) {
switch (idx) {
- case ExchangeType::NOOP:
+ case TLocalPartitionType::NOOP:
return "NOOP";
- case ExchangeType::HASH_SHUFFLE:
- return "HASH_SHUFFLE";
- case ExchangeType::PASSTHROUGH:
+ case TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE:
+ return "GLOBAL_HASH_SHUFFLE";
+ case TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE:
+ return "LOCAL_HASH_SHUFFLE";
+ case TLocalPartitionType::PASSTHROUGH:
return "PASSTHROUGH";
- case ExchangeType::BUCKET_HASH_SHUFFLE:
+ case TLocalPartitionType::BUCKET_HASH_SHUFFLE:
return "BUCKET_HASH_SHUFFLE";
- case ExchangeType::BROADCAST:
+ case TLocalPartitionType::BROADCAST:
return "BROADCAST";
- case ExchangeType::ADAPTIVE_PASSTHROUGH:
+ case TLocalPartitionType::ADAPTIVE_PASSTHROUGH:
return "ADAPTIVE_PASSTHROUGH";
- case ExchangeType::PASS_TO_ONE:
+ case TLocalPartitionType::PASS_TO_ONE:
return "PASS_TO_ONE";
+ case TLocalPartitionType::LOCAL_MERGE_SORT:
+ return "LOCAL_MERGE_SORT";
}
throw Exception(Status::FatalError("__builtin_unreachable"));
}
struct DataDistribution {
- DataDistribution(ExchangeType type) : distribution_type(type) {}
- DataDistribution(ExchangeType type, const std::vector<TExpr>& partition_exprs_)
+ DataDistribution(TLocalPartitionType::type type) : distribution_type(type) {}
+ DataDistribution(TLocalPartitionType::type type, const std::vector<TExpr>& partition_exprs_)
: distribution_type(type), partition_exprs(partition_exprs_) {}
DataDistribution(const DataDistribution& other) = default;
- bool need_local_exchange() const { return distribution_type != ExchangeType::NOOP; }
+ bool need_local_exchange() const { return distribution_type != TLocalPartitionType::NOOP; }
DataDistribution& operator=(const DataDistribution& other) = default;
- ExchangeType distribution_type;
+ TLocalPartitionType::type distribution_type;
std::vector<TExpr> partition_exprs;
};
diff --git a/be/src/exec/pipeline/pipeline.cpp b/be/src/exec/pipeline/pipeline.cpp
index de3c852ada1..c63267800ed 100644
--- a/be/src/exec/pipeline/pipeline.cpp
+++ b/be/src/exec/pipeline/pipeline.cpp
@@ -54,8 +54,7 @@ bool Pipeline::need_to_local_exchange(const DataDistribution target_data_distrib
return true;
}
- if (target_data_distribution.distribution_type != ExchangeType::BUCKET_HASH_SHUFFLE &&
- target_data_distribution.distribution_type != ExchangeType::HASH_SHUFFLE) {
+ if (!is_shuffled_exchange(target_data_distribution.distribution_type)) {
// Always do local exchange if non-hash-partition exchanger is required.
// For example, `PASSTHROUGH` exchanger is always required to distribute data evenly.
return true;
diff --git a/be/src/exec/pipeline/pipeline.h b/be/src/exec/pipeline/pipeline.h
index 3dc9e2353f7..817c379692e 100644
--- a/be/src/exec/pipeline/pipeline.h
+++ b/be/src/exec/pipeline/pipeline.h
@@ -68,16 +68,15 @@ public:
[[nodiscard]] PipelineId id() const { return _pipeline_id; }
- static bool is_hash_exchange(ExchangeType idx) {
- return idx == ExchangeType::HASH_SHUFFLE || idx == ExchangeType::BUCKET_HASH_SHUFFLE;
+ static bool is_hash_exchange(TLocalPartitionType::type idx) {
+ return is_shuffled_exchange(idx);
}
// For HASH_SHUFFLE, BUCKET_HASH_SHUFFLE, and ADAPTIVE_PASSTHROUGH,
// data is processed and shuffled on the sink.
// Compared to PASSTHROUGH, this is a relatively heavy operation.
- static bool heavy_operations_on_the_sink(ExchangeType idx) {
- return idx == ExchangeType::HASH_SHUFFLE || idx == ExchangeType::BUCKET_HASH_SHUFFLE ||
- idx == ExchangeType::ADAPTIVE_PASSTHROUGH;
+ static bool heavy_operations_on_the_sink(TLocalPartitionType::type idx) {
+ return is_shuffled_exchange(idx) || idx == TLocalPartitionType::ADAPTIVE_PASSTHROUGH;
}
bool need_to_local_exchange(const DataDistribution target_data_distribution,
@@ -165,7 +164,7 @@ private:
// Input data distribution of this pipeline. We do local exchange when input data distribution
// does not match the target data distribution.
- DataDistribution _data_distribution {ExchangeType::NOOP};
+ DataDistribution _data_distribution {TLocalPartitionType::NOOP};
// How many tasks should be created ?
int _num_tasks = 1;
diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp
index 8570c771232..cac696397c1 100644
--- a/be/src/exec/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp
@@ -290,6 +290,32 @@ Status PipelineFragmentContext::_build_and_prepare_full_pipeline(ThreadPool* thr
RETURN_IF_ERROR(_build_pipelines(_runtime_state->obj_pool(), *_query_ctx->desc_tbl,
&_root_op, root_pipeline));
+ // Propagate _num_instances from LOCAL_EXCHANGE pipelines to ancestor pipelines
+ // that inherited reduced num_tasks from a serial operator.
+ _propagate_local_exchange_num_tasks();
+
+ // Create deferred local exchangers now that all pipelines have final num_tasks.
+ RETURN_IF_ERROR(_create_deferred_local_exchangers());
+
+ // Raise num_tasks for pipelines whose serial non-scan operators (e.g.,
+ // UNPARTITIONED Exchange) reduced num_tasks below _num_instances.
+ // Without this, fragment instances 1+ have no task for these pipelines
+ // and downstream operators fail with "must set shared state".
+ //
+ // This applies to ALL pipelines (not just deferred exchanger upstreams):
+ // fragments with UNION/INTERSECT/EXCEPT + serial Exchange in child
+ // pipelines also need the raise, even without FE-planned local exchange.
+ //
+ // Exception: serial scan sources (pooling scan) keep num_tasks=1 — the
+ // PassthroughExchanger(1, N) handles the fan-out correctly.
+ // NOTE: Do NOT raise pipelines whose source is a serial operator
+ // (Exchange or scan) — they legitimately have 1 task, and raising
+ // them causes crashes (e.g., 4 Exchange tasks but only 1 receives
+ // data). The correct fix for shared state injection across
+ // instances is handled by the FE: it inserts local exchange nodes
+ // between serial operators and their downstream consumers, creating
+ // proper pipeline boundaries with _num_instances tasks.
+
// 3. Create sink operator
if (!_params.fragment.__isset.output_sink) {
return Status::InternalError("No output sink in this fragment!");
@@ -307,7 +333,7 @@ Status PipelineFragmentContext::_build_and_prepare_full_pipeline(ThreadPool* thr
}
}
// 4. Build local exchanger
- if (_runtime_state->enable_local_shuffle()) {
+ if (_runtime_state->plan_local_shuffle()) {
SCOPED_TIMER(_plan_local_exchanger_timer);
RETURN_IF_ERROR(_plan_local_exchange(_params.num_buckets,
_params.bucket_seq_to_instance_idx,
@@ -690,6 +716,150 @@ Status PipelineFragmentContext::_build_pipelines(ObjectPool* pool, const Descrip
return Status::OK();
}
+Status PipelineFragmentContext::_create_deferred_local_exchangers() {
+ for (auto& info : _deferred_exchangers) {
+ const int sender_count = info.upstream_pipe->num_tasks();
+ switch (info.partition_type) {
+ case TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE:
+ case TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE:
+ info.shared_state->exchanger = ShuffleExchanger::create_unique(
+ sender_count, _num_instances, info.num_partitions, info.free_blocks_limit,
+ info.partition_type);
+ break;
+ case TLocalPartitionType::BUCKET_HASH_SHUFFLE:
+ info.shared_state->exchanger = BucketShuffleExchanger::create_unique(
+ sender_count, _num_instances, info.num_partitions, info.free_blocks_limit);
+ break;
+ case TLocalPartitionType::PASSTHROUGH:
+ info.shared_state->exchanger = PassthroughExchanger::create_unique(
+ sender_count, _num_instances, info.free_blocks_limit);
+ break;
+ case TLocalPartitionType::BROADCAST:
+ info.shared_state->exchanger = BroadcastExchanger::create_unique(
+ sender_count, _num_instances, info.free_blocks_limit);
+ break;
+ case TLocalPartitionType::PASS_TO_ONE:
+ if (_runtime_state->enable_share_hash_table_for_broadcast_join()) {
+ info.shared_state->exchanger = PassToOneExchanger::create_unique(
+ sender_count, _num_instances, info.free_blocks_limit);
+ } else {
+ info.shared_state->exchanger = BroadcastExchanger::create_unique(
+ sender_count, _num_instances, info.free_blocks_limit);
+ }
+ break;
+ case TLocalPartitionType::ADAPTIVE_PASSTHROUGH:
+ info.shared_state->exchanger = AdaptivePassthroughExchanger::create_unique(
+ sender_count, _num_instances, info.free_blocks_limit);
+ break;
+ default:
+ return Status::InternalError("Unsupported FE-planned local exchange type: " +
+ std::to_string(static_cast<int>(info.partition_type)));
+ }
+ }
+ _deferred_exchangers.clear();
+ return Status::OK();
+}
+
+void PipelineFragmentContext::_propagate_local_exchange_num_tasks() {
+ if (_deferred_exchangers.empty()) {
+ return;
+ }
+ // Fix num_tasks mismatches between paired pipelines created by pipeline-splitting
+ // operators (AGG, SORT, JOIN). These pairs share state via inject_shared_state and
+ // must have consistent num_tasks; otherwise instance 1+ tasks access null shared_state.
+ //
+ // Pass 1 (upward): when FE inserts LOCAL_EXCHANGE_NODE below a splitting operator,
+ // the LE pipeline gets _num_instances but ancestor pipelines still have the reduced
+ // count. Raise them to _num_instances (skip serial source operators).
+ //
+ // Pass 2 (downward): when a serial Exchange feeds a splitting operator (e.g., scalar
+ // merge-finalize AGG), the sink pipeline has 1 task but the source pipeline may have
+ // been raised (by LE or by inheriting _num_instances from add_pipeline). Lower it
+ // to match, unless it contains LocalExchangeSource (deliberately raised by LE).
+ std::unordered_map<PipelineId, PipelinePtr> id_to_pipe;
+ for (auto& p : _pipelines) {
+ id_to_pipe[p->id()] = p;
+ }
+
+ // Pass 1 (upward): raise downstream pipelines to _num_instances when any upstream
+ // dependency was already raised by LOCAL_EXCHANGE.
+ bool changed = true;
+ while (changed) {
+ changed = false;
+ for (const auto& [downstream_id, upstream_ids] : _dag) {
+ auto dit = id_to_pipe.find(downstream_id);
+ if (dit == id_to_pipe.end()) {
+ continue;
+ }
+ auto& downstream = dit->second;
+ if (downstream->num_tasks() >= _num_instances) {
+ continue;
+ }
+ if (!downstream->operators().empty() &&
+ downstream->operators().front()->is_serial_operator()) {
+ continue;
+ }
+ for (auto upstream_id : upstream_ids) {
+ auto uit = id_to_pipe.find(upstream_id);
+ if (uit != id_to_pipe.end() && uit->second->num_tasks() >= _num_instances) {
+ downstream->set_num_tasks(_num_instances);
+ changed = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Pass 2 (downward): when a pipeline-splitting operator (AGG, SORT, JOIN) creates
+ // paired pipelines (source + sink), they share state via inject_shared_state and must
+ // have consistent num_tasks. If the sink pipeline has a serial source (Exchange) with
+ // num_tasks < _num_instances, but the source pipeline was raised (by inheriting from
+ // an LE-fixed parent during add_pipeline), lower the source pipeline to match.
+ // Skip pipelines containing LocalExchangeSource — they were deliberately raised.
+ //
+ // Example: serial Exchange → AGG(merge finalize) → LOCAL_EXCHANGE → NLJ
+ // P_sink = [Exchange, AGGSink] num_tasks=1 (serial)
+ // P_source = [AGGSource] num_tasks=_num_instances (inherited)
+ // → lower P_source to 1; the LE above fans out the 1 result to N NLJ tasks.
+ changed = true;
+ while (changed) {
+ changed = false;
+ for (const auto& [downstream_id, upstream_ids] : _dag) {
+ auto dit = id_to_pipe.find(downstream_id);
+ if (dit == id_to_pipe.end()) {
+ continue;
+ }
+ auto& downstream = dit->second;
+ if (downstream->num_tasks() <= 1) {
+ continue;
+ }
+ // Don't lower pipelines that contain LocalExchangeSource — they were
+ // deliberately raised to _num_instances by LOCAL_EXCHANGE processing.
+ bool has_le_source = false;
+ for (auto& op : downstream->operators()) {
+ if (dynamic_cast<LocalExchangeSourceOperatorX*>(op.get())) {
+ has_le_source = true;
+ break;
+ }
+ }
+ if (has_le_source) {
+ continue;
+ }
+ for (auto upstream_id : upstream_ids) {
+ auto uit = id_to_pipe.find(upstream_id);
+ if (uit != id_to_pipe.end() &&
+ uit->second->num_tasks() < downstream->num_tasks() &&
+ !uit->second->operators().empty() &&
+ uit->second->operators().front()->is_serial_operator()) {
+ downstream->set_num_tasks(uit->second->num_tasks());
+ changed = true;
+ break;
+ }
+ }
+ }
+ }
+}
+
Status PipelineFragmentContext::_create_tree_helper(
ObjectPool* pool, const std::vector<TPlanNode>& tnodes, const DescriptorTbl& descs,
OperatorPtr parent, int* node_idx, OperatorPtr* root, PipelinePtr& cur_pipe, int child_idx,
@@ -723,7 +893,7 @@ Status PipelineFragmentContext::_create_tree_helper(
*root = op;
}
/**
- * `ExchangeType::HASH_SHUFFLE` should be used if an operator is followed by a shuffled operator (shuffled hash join, union operator followed by co-located operators).
+ * `TLocalPartitionType::HASH_SHUFFLE` should be used if an operator is followed by a shuffled operator (shuffled hash join, union operator followed by co-located operators).
*
* For plan:
* LocalExchange(id=0) -> Aggregation(id=1) -> ShuffledHashJoin(id=2)
@@ -743,7 +913,7 @@ Status PipelineFragmentContext::_create_tree_helper(
: op->is_shuffled_operator())) &&
Pipeline::is_hash_exchange(required_data_distribution.distribution_type)) ||
(followed_by_shuffled_operator &&
- required_data_distribution.distribution_type == ExchangeType::NOOP);
+ required_data_distribution.distribution_type == TLocalPartitionType::NOOP);
current_require_bucket_distribution =
((require_bucket_distribution ||
@@ -751,7 +921,7 @@ Status PipelineFragmentContext::_create_tree_helper(
: op->is_colocated_operator())) &&
Pipeline::is_hash_exchange(required_data_distribution.distribution_type)) ||
(require_bucket_distribution &&
- required_data_distribution.distribution_type == ExchangeType::NOOP);
+ required_data_distribution.distribution_type == TLocalPartitionType::NOOP);
if (num_children == 0) {
_use_serial_source = op->is_serial_operator();
@@ -810,28 +980,35 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
sink_id, local_exchange_id, use_global_hash_shuffle ? _total_instances : _num_instances,
data_distribution.partition_exprs, bucket_seq_to_instance_idx);
if (bucket_seq_to_instance_idx.empty() &&
- data_distribution.distribution_type == ExchangeType::BUCKET_HASH_SHUFFLE) {
- data_distribution.distribution_type = ExchangeType::HASH_SHUFFLE;
+ data_distribution.distribution_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) {
+ data_distribution.distribution_type =
+ use_global_hash_shuffle ? TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE
+ : TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE;
+ }
+ if (!use_global_hash_shuffle &&
+ data_distribution.distribution_type == TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE) {
+ data_distribution.distribution_type = TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE;
}
RETURN_IF_ERROR(new_pip->set_sink(sink));
RETURN_IF_ERROR(new_pip->sink()->init(_runtime_state.get(), data_distribution.distribution_type,
- num_buckets, use_global_hash_shuffle,
- shuffle_idx_to_instance_idx));
+ num_buckets, shuffle_idx_to_instance_idx));
// 2. Create and initialize LocalExchangeSharedState.
std::shared_ptr<LocalExchangeSharedState> shared_state =
LocalExchangeSharedState::create_shared(_num_instances);
switch (data_distribution.distribution_type) {
- case ExchangeType::HASH_SHUFFLE:
+ case TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE:
+ case TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE:
shared_state->exchanger = ShuffleExchanger::create_unique(
std::max(cur_pipe->num_tasks(), _num_instances), _num_instances,
use_global_hash_shuffle ? _total_instances : _num_instances,
_runtime_state->query_options().__isset.local_exchange_free_blocks_limit
? cast_set<int>(
_runtime_state->query_options().local_exchange_free_blocks_limit)
- : 0);
+ : 0,
+ data_distribution.distribution_type);
break;
- case ExchangeType::BUCKET_HASH_SHUFFLE:
+ case TLocalPartitionType::BUCKET_HASH_SHUFFLE:
shared_state->exchanger = BucketShuffleExchanger::create_unique(
std::max(cur_pipe->num_tasks(), _num_instances), _num_instances, num_buckets,
_runtime_state->query_options().__isset.local_exchange_free_blocks_limit
@@ -839,7 +1016,7 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
_runtime_state->query_options().local_exchange_free_blocks_limit)
: 0);
break;
- case ExchangeType::PASSTHROUGH:
+ case TLocalPartitionType::PASSTHROUGH:
shared_state->exchanger = PassthroughExchanger::create_unique(
cur_pipe->num_tasks(), _num_instances,
_runtime_state->query_options().__isset.local_exchange_free_blocks_limit
@@ -847,7 +1024,7 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
_runtime_state->query_options().local_exchange_free_blocks_limit)
: 0);
break;
- case ExchangeType::BROADCAST:
+ case TLocalPartitionType::BROADCAST:
shared_state->exchanger = BroadcastExchanger::create_unique(
cur_pipe->num_tasks(), _num_instances,
_runtime_state->query_options().__isset.local_exchange_free_blocks_limit
@@ -855,7 +1032,7 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
_runtime_state->query_options().local_exchange_free_blocks_limit)
: 0);
break;
- case ExchangeType::PASS_TO_ONE:
+ case TLocalPartitionType::PASS_TO_ONE:
if (_runtime_state->enable_share_hash_table_for_broadcast_join()) {
// If shared hash table is enabled for BJ, hash table will be built by only one task
shared_state->exchanger = PassToOneExchanger::create_unique(
@@ -873,7 +1050,7 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
: 0);
}
break;
- case ExchangeType::ADAPTIVE_PASSTHROUGH:
+ case TLocalPartitionType::ADAPTIVE_PASSTHROUGH:
shared_state->exchanger = AdaptivePassthroughExchanger::create_unique(
std::max(cur_pipe->num_tasks(), _num_instances), _num_instances,
_runtime_state->query_options().__isset.local_exchange_free_blocks_limit
@@ -984,9 +1161,9 @@ Status PipelineFragmentContext::_add_local_exchange(
Pipeline::heavy_operations_on_the_sink(data_distribution.distribution_type)) {
RETURN_IF_ERROR(_add_local_exchange_impl(
cast_set<int>(new_pip->operators().size()), pool, new_pip,
- add_pipeline(new_pip, pip_idx + 2), DataDistribution(ExchangeType::PASSTHROUGH),
- do_local_exchange, num_buckets, bucket_seq_to_instance_idx,
- shuffle_idx_to_instance_idx));
+ add_pipeline(new_pip, pip_idx + 2),
+ DataDistribution(TLocalPartitionType::PASSTHROUGH), do_local_exchange, num_buckets,
+ bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx));
}
return Status::OK();
}
@@ -1027,10 +1204,11 @@ Status PipelineFragmentContext::_plan_local_exchange(
do_local_exchange = false;
// Plan local exchange for each operator.
for (; idx < ops.size();) {
- if (ops[idx]->required_data_distribution(_runtime_state.get()).need_local_exchange()) {
+ auto _le_req = ops[idx]->required_data_distribution(_runtime_state.get());
+ if (_le_req.need_local_exchange()) {
RETURN_IF_ERROR(_add_local_exchange(
pip_idx, idx, ops[idx]->node_id(), _runtime_state->obj_pool(), pip,
- ops[idx]->required_data_distribution(_runtime_state.get()),
+ _le_req,
&do_local_exchange, num_buckets, bucket_seq_to_instance_idx,
shuffle_idx_to_instance_idx));
}
@@ -1796,6 +1974,89 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo
RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances));
break;
}
+ case TPlanNodeType::LOCAL_EXCHANGE_NODE: {
+ op = std::make_shared<LocalExchangeSourceOperatorX>(pool, tnode, next_operator_id(), descs);
+ // The downstream pipeline (containing LocalExchangeSource) must have
+ // _num_instances tasks — matching BE-native _inherit_pipeline_properties
+ // which sets pipe_with_source.set_num_tasks(_num_instances).
+ // Without this, when the parent pipeline was reduced by a serial operator
+ // (e.g., serial Exchange with use_serial_exchange=true, or UNPARTITIONED
+ // Exchange), the downstream inherits the reduced num_tasks via
+ // add_pipeline(parent). The deferred exchanger creates _num_instances
+ // channels but only fewer source tasks initialize mem_counters — the
+ // sink round-robins to all channels and crashes on uninitialized ones.
+ auto downstream_num_tasks = _num_instances;
+ RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances));
+ // Restore downstream pipeline's num_tasks (mirroring _inherit_pipeline_properties:
+ // downstream keeps _num_instances, upstream gets the serial/reduced count)
+ cur_pipe->set_num_tasks(downstream_num_tasks);
+
+ const auto downstream_pipeline_id = cur_pipe->id();
+ if (!_dag.contains(downstream_pipeline_id)) {
+ _dag.insert({downstream_pipeline_id, {}});
+ }
+ cur_pipe = add_pipeline(cur_pipe);
+ // If this local exchange was inserted because of a serial scan (is_serial_operator),
+ // the upstream pipeline (cur_pipe) should have num_tasks=1 (only 1 scan task).
+ // We set this now so the exchanger is created with the correct sender count.
+ // Child operators added later (serial scan) will also set num_tasks=1, which is
+ // consistent with this.
+ if (op->is_serial_operator() && _parallel_instances > 0) {
+ cur_pipe->set_num_tasks(_parallel_instances);
+ }
+ _dag[downstream_pipeline_id].push_back(cur_pipe->id());
+ int num_partitions = 0;
+ std::map<int, int> shuffle_id_to_instance_idx;
+ auto partition_type = tnode.local_exchange_node.partition_type;
+ switch (partition_type) {
+ case TLocalPartitionType::BUCKET_HASH_SHUFFLE:
+ num_partitions = _params.num_buckets;
+ shuffle_id_to_instance_idx = _params.bucket_seq_to_instance_idx;
+ break;
+ case TLocalPartitionType::LOCAL_EXECUTION_HASH_SHUFFLE:
+ for (int i = 0; i < _num_instances; i++) {
+ shuffle_id_to_instance_idx[i] = i;
+ }
+ num_partitions = _num_instances;
+ break;
+ case TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE:
+ num_partitions = _total_instances;
+ shuffle_id_to_instance_idx = _params.shuffle_idx_to_instance_idx;
+ break;
+ default:
+ break;
+ }
+ auto local_exchange_id = op->operator_id();
+ auto sink_id = next_sink_operator_id();
+ DataSinkOperatorPtr sink = std::make_shared<LocalExchangeSinkOperatorX>(
+ sink_id, local_exchange_id, tnode, num_partitions, shuffle_id_to_instance_idx);
+ sink_ops.push_back(sink);
+ RETURN_IF_ERROR(cur_pipe->set_sink(sink));
+ RETURN_IF_ERROR(cur_pipe->sink()->init(tnode, _runtime_state.get()));
+
+ // For FE-planned local exchange, we need to:
+ // 1. Initialize the partitioner for hash shuffle types
+ // 2. Defer exchanger creation until after the full plan tree is built
+ // (child operators like serial ExchangeNode may change cur_pipe->num_tasks())
+ // 3. Register shared state so pipeline tasks can find it
+ RETURN_IF_ERROR(static_cast<LocalExchangeSinkOperatorX*>(cur_pipe->sink())
+ ->init_partitioner(_runtime_state.get()));
+
+ int free_blocks_limit =
+ _runtime_state->query_options().__isset.local_exchange_free_blocks_limit
+ ? cast_set<int>(
+ _runtime_state->query_options().local_exchange_free_blocks_limit)
+ : 0;
+ auto shared_state = LocalExchangeSharedState::create_shared(_num_instances);
+ shared_state->create_source_dependencies(_num_instances, local_exchange_id,
+ local_exchange_id, "LOCAL_EXCHANGE_OPERATOR");
+ shared_state->create_sink_dependency(sink_id, local_exchange_id, "LOCAL_EXCHANGE_SINK");
+ _op_id_to_shared_state.insert({local_exchange_id, {shared_state, shared_state->sink_deps}});
+ // Defer exchanger creation: sender count depends on final upstream num_tasks
+ _deferred_exchangers.push_back({shared_state, cur_pipe, partition_type, num_partitions,
+ free_blocks_limit, local_exchange_id, sink_id});
+ break;
+ }
default:
return Status::InternalError("Unsupported exec type in pipeline: {}",
print_plan_node_type(tnode.node_type));
diff --git a/be/src/exec/pipeline/pipeline_fragment_context.h b/be/src/exec/pipeline/pipeline_fragment_context.h
index c220ea386f6..4950da0a7b0 100644
--- a/be/src/exec/pipeline/pipeline_fragment_context.h
+++ b/be/src/exec/pipeline/pipeline_fragment_context.h
@@ -18,6 +18,7 @@
#pragma once
#include <brpc/closure_guard.h>
+#include <gen_cpp/Partitions_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/types.pb.h>
@@ -49,6 +50,7 @@ class TDataSink;
class TPipelineFragmentParams;
class Dependency;
+struct LocalExchangeSharedState;
class PipelineFragmentContext : public TaskExecutionContext {
public:
@@ -320,8 +322,11 @@ private:
std::mutex _state_map_lock;
- int _operator_id = 0;
- int _sink_operator_id = 0;
+ // Start from -1 so all operator IDs are negative. This avoids collision with
+ // unpaired sinks (OlapTableSink etc.) whose hardcoded dest_id=0 would otherwise
+ // match the first operator's ID when FE-planned LocalExchangeNode is the root.
+ int _operator_id = -1;
+ int _sink_operator_id = -1;
/**
* Some states are shared by tasks in different pipeline task (e.g. local exchange , broadcast join).
*
@@ -342,6 +347,27 @@ private:
std::map<PipelineId, Pipeline*> _pip_id_to_pipeline;
std::vector<std::unique_ptr<RuntimeFilterMgr>> _runtime_filter_mgr_map;
+ // Deferred exchanger creation info for FE-planned local exchanges.
+ // Exchanger sender count depends on the upstream pipeline's final num_tasks,
+ // which is only known after the full plan tree is built (child operators like
+ // serial ExchangeNode may reduce num_tasks). So we defer exchanger creation
+ // until after _build_pipelines completes.
+ struct DeferredExchangerInfo {
+ std::shared_ptr<LocalExchangeSharedState> shared_state;
+ PipelinePtr upstream_pipe;
+ TLocalPartitionType::type partition_type;
+ int num_partitions;
+ int free_blocks_limit;
+ int local_exchange_id;
+ int sink_id;
+ };
+ std::vector<DeferredExchangerInfo> _deferred_exchangers;
+ Status _create_deferred_local_exchangers();
+ // After _build_pipelines, propagate _num_instances from FE-planned LOCAL_EXCHANGE
+ // pipelines upward through the DAG to ancestor pipelines that inherited reduced
+ // num_tasks from a serial operator.
+ void _propagate_local_exchange_num_tasks();
+
//Here are two types of runtime states:
// - _runtime state is at the Fragment level.
// - _task_runtime_states is at the task level, unique to each task.
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 94658e9e80e..ae461e2c8dc 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -627,6 +627,17 @@ Status ExecEnv::init_mem_env() {
} else {
fd_number = static_cast<uint64_t>(l.rlim_cur);
}
+#ifdef __APPLE__
+ // On macOS, rlim_cur can be RLIM_INFINITY (INT64_MAX), which causes
+ // fd_number / 100 * percentage to overflow and crash cast_set<uint32_t>.
+ // Linux kernels cap this via fs.nr_open (default 1M), so only macOS needs this.
+ {
+ constexpr uint64_t max_fd = UINT32_MAX >> 2;
+ if (fd_number > max_fd) {
+ fd_number = max_fd;
+ }
+ }
+#endif
int64_t segment_cache_capacity = 0;
if (config::is_cloud_mode()) {
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 905b1ffa611..bc1124cb18a 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -421,8 +421,11 @@ public:
BeExecVersionManager::check_be_exec_version(_query_options.be_exec_version));
return _query_options.be_exec_version;
}
- bool enable_local_shuffle() const {
- return _query_options.__isset.enable_local_shuffle && _query_options.enable_local_shuffle;
+ bool plan_local_shuffle() const {
+ // If local shuffle is enabled and not planned by local shuffle planner, we should plan local shuffle in BE.
+ return _query_options.__isset.enable_local_shuffle && _query_options.enable_local_shuffle &&
+ (!_query_options.__isset.enable_local_shuffle_planner ||
+ !_query_options.enable_local_shuffle_planner);
}
MOCK_FUNCTION bool enable_local_exchange() const {
diff --git a/be/src/udf/python/python_udf_runtime.cpp b/be/src/udf/python/python_udf_runtime.cpp
index 9d687c43f40..3fa94bf417e 100644
--- a/be/src/udf/python/python_udf_runtime.cpp
+++ b/be/src/udf/python/python_udf_runtime.cpp
@@ -21,8 +21,6 @@
#include <sys/wait.h>
#include <unistd.h>
-#include <boost/process.hpp>
-
#include "common/logging.h"
namespace doris {
diff --git a/be/test/exec/pipeline/local_exchanger_test.cpp b/be/test/exec/pipeline/local_exchanger_test.cpp
index 3051625a3ee..22dacde4568 100644
--- a/be/test/exec/pipeline/local_exchanger_test.cpp
+++ b/be/test/exec/pipeline/local_exchanger_test.cpp
@@ -89,8 +89,9 @@ TEST_F(LocalExchangerTest, ShuffleExchanger) {
_local_states.resize(num_sources);
auto profile = std::make_shared<RuntimeProfile>("");
auto shared_state = LocalExchangeSharedState::create_shared(num_partitions);
- shared_state->exchanger = ShuffleExchanger::create_unique(num_sink, num_sources, num_partitions,
- free_block_limit);
+ shared_state->exchanger =
+ ShuffleExchanger::create_unique(num_sink, num_sources, num_partitions, free_block_limit,
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
auto sink_dep = std::make_shared<Dependency>(0, 0, "LOCAL_EXCHANGE_SINK_DEPENDENCY", true);
sink_dep->set_shared_state(shared_state.get());
shared_state->sink_deps.push_back(sink_dep);
@@ -1175,8 +1176,9 @@ TEST_F(LocalExchangerTest, TestShuffleExchangerWrongMap) {
_local_states.resize(num_sources);
auto profile = std::make_shared<RuntimeProfile>("");
auto shared_state = LocalExchangeSharedState::create_shared(num_partitions);
- shared_state->exchanger = ShuffleExchanger::create_unique(num_sink, num_sources, num_partitions,
- free_block_limit);
+ shared_state->exchanger =
+ ShuffleExchanger::create_unique(num_sink, num_sources, num_partitions, free_block_limit,
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
auto sink_dep = std::make_shared<Dependency>(0, 0, "LOCAL_EXCHANGE_SINK_DEPENDENCY", true);
sink_dep->set_shared_state(shared_state.get());
shared_state->sink_deps.push_back(sink_dep);
diff --git a/be/test/exec/pipeline/pipeline_test.cpp b/be/test/exec/pipeline/pipeline_test.cpp
index d5d0b5028d6..714d668bffc 100644
--- a/be/test/exec/pipeline/pipeline_test.cpp
+++ b/be/test/exec/pipeline/pipeline_test.cpp
@@ -509,7 +509,7 @@ TEST_F(PipelineTest, PLAN_LOCAL_EXCHANGE) {
DescriptorTbl* desc;
OperatorPtr op;
_build_fragment_context();
- EXPECT_EQ(_runtime_state.front()->enable_local_shuffle(), true);
+ EXPECT_EQ(_runtime_state.front()->plan_local_shuffle(), true);
auto cur_pipe = _build_pipeline(parallelism);
{
auto tnode = TPlanNodeBuilder(_next_node_id(), TPlanNodeType::EXCHANGE_NODE)
@@ -584,11 +584,12 @@ TEST_F(PipelineTest, PLAN_LOCAL_EXCHANGE) {
}
{
cur_pipe->init_data_distribution(_runtime_state.back().get());
- EXPECT_EQ(cur_pipe->data_distribution().distribution_type, ExchangeType::HASH_SHUFFLE);
+ EXPECT_EQ(cur_pipe->data_distribution().distribution_type,
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
EXPECT_EQ(cur_pipe->sink()
->required_data_distribution(_runtime_state.back().get())
.distribution_type,
- ExchangeType::NOOP);
+ TLocalPartitionType::NOOP);
EXPECT_EQ(cur_pipe->need_to_local_exchange(
cur_pipe->sink()->required_data_distribution(_runtime_state.back().get()),
1),
@@ -597,11 +598,11 @@ TEST_F(PipelineTest, PLAN_LOCAL_EXCHANGE) {
{
cur_pipe->operators().front()->set_serial_operator();
cur_pipe->init_data_distribution(_runtime_state.back().get());
- EXPECT_EQ(cur_pipe->data_distribution().distribution_type, ExchangeType::NOOP);
+ EXPECT_EQ(cur_pipe->data_distribution().distribution_type, TLocalPartitionType::NOOP);
EXPECT_EQ(cur_pipe->sink()
->required_data_distribution(_runtime_state.back().get())
.distribution_type,
- ExchangeType::PASSTHROUGH);
+ TLocalPartitionType::PASSTHROUGH);
EXPECT_EQ(cur_pipe->need_to_local_exchange(
cur_pipe->sink()->required_data_distribution(_runtime_state.back().get()),
1),
@@ -620,7 +621,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) {
// Build pipeline
DescriptorTbl* desc;
_build_fragment_context();
- EXPECT_EQ(_runtime_state.front()->enable_local_shuffle(), true);
+ EXPECT_EQ(_runtime_state.front()->plan_local_shuffle(), true);
{
TTupleDescriptor tuple0 = TTupleDescriptorBuilder().set_id(0).build();
TSlotDescriptor slot0 =
@@ -903,12 +904,12 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) {
if (pip_idx == 1) {
// Pipeline(ExchangeOperator(id=1, HASH_PARTITIONED) -> HashJoinBuildOperator(id=0))
EXPECT_EQ(_pipelines[pip_idx]->data_distribution().distribution_type,
- ExchangeType::HASH_SHUFFLE);
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
EXPECT_EQ(_pipelines[pip_idx]
->sink()
->required_data_distribution(_runtime_state.back().get())
.distribution_type,
- ExchangeType::HASH_SHUFFLE);
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
EXPECT_EQ(_pipelines[pip_idx]->need_to_local_exchange(
_pipelines[pip_idx]->sink()->required_data_distribution(
_runtime_state.back().get()),
@@ -919,7 +920,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) {
_pipelines[pip_idx]->set_data_distribution(
_pipelines[pip_idx]->children().front()->data_distribution());
EXPECT_EQ(_pipelines[pip_idx]->data_distribution().distribution_type,
- ExchangeType::HASH_SHUFFLE);
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
EXPECT_EQ(_pipelines[pip_idx]->need_to_local_exchange(
_pipelines[pip_idx]->sink()->required_data_distribution(
_runtime_state.back().get()),
@@ -930,7 +931,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) {
.back()
->required_data_distribution(_runtime_state.back().get())
.distribution_type,
- ExchangeType::HASH_SHUFFLE);
+ TLocalPartitionType::GLOBAL_EXECUTION_HASH_SHUFFLE);
EXPECT_EQ(_pipelines[pip_idx]->need_to_local_exchange(
_pipelines[pip_idx]->operators().back()->required_data_distribution(
_runtime_state.back().get()),
diff --git a/fe/.idea/vcs.xml b/fe/.idea/vcs.xml
index e5158c553f8..c1644214b8a 100644
--- a/fe/.idea/vcs.xml
+++ b/fe/.idea/vcs.xml
@@ -12,6 +12,11 @@
</component>
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/../.github/actions/action-pr-title" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/../.github/actions/ccache-action" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/../.github/actions/get-workflow-origin" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/../.github/actions/label-when-approved-action" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/../.github/actions/paths-filter" vcs="Git" />
<mapping directory="$PROJECT_DIR$/../contrib/apache-orc" vcs="Git" />
<mapping directory="$PROJECT_DIR$/../contrib/clucene" vcs="Git" />
<mapping directory="$PROJECT_DIR$/../contrib/faiss" vcs="Git" />
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index 3ad317792a5..7a32d339f4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -113,6 +113,10 @@ public class SummaryProfile {
public static final String READ_BYTES_PER_SECOND = "Read Bytes Per Second";
public static final String REMOTE_READ_BYTES_PER_SECOND = "Remote Read Bytes Per Second";
+ // Written last in queryFinished(), after all BE fragment profiles are merged.
+ // Readers can poll for this field to know the profile is fully collected.
+ public static final String IS_PROFILE_COLLECTION_COMPLETED = "Is Profile Collection Completed";
+
public static final String PARSE_SQL_TIME = "Parse SQL Time";
public static final String NEREIDS_LOCK_TABLE_TIME = "Nereids Lock Table Time";
public static final String NEREIDS_ANALYSIS_TIME = "Nereids Analysis Time";
@@ -514,6 +518,10 @@ public class SummaryProfile {
// This method is used to display the final data status when the overall query ends.
// This can avoid recalculating some strings and so on every time during the update process.
public void queryFinished() {
+ // Mark profile collection as complete. This is written last, after all BE fragment
+ // profiles have been merged (called post-waitForFragmentsDone). Pollers can wait for
+ // this field to avoid reading a partial profile.
+ executionSummaryProfile.addInfoString(IS_PROFILE_COLLECTION_COMPLETED, "true");
if (assignedWeightPerBackend != null) {
Map<String, Long> m = assignedWeightPerBackend.entrySet().stream()
.sorted(Map.Entry.comparingByValue())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index ce22e96c80b..644bc0504ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.FunctionGenTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
@@ -33,6 +34,10 @@ import org.apache.doris.datasource.FileSplit;
import org.apache.doris.datasource.FileSplit.FileSplitCreator;
import org.apache.doris.datasource.FileSplitter;
import org.apache.doris.datasource.TableFormatType;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
+import org.apache.doris.planner.PlanNode;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanContext;
import org.apache.doris.qe.SessionVariable;
@@ -212,4 +217,10 @@ public class TVFScanNode extends FileQueryScanNode {
public int getNumInstances() {
return scanRangeLocations.size();
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ return Pair.of(this, LocalExchangeType.PASSTHROUGH);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 7952d009e51..b0827116e84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -75,6 +75,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache;
import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
+import org.apache.doris.planner.AddLocalExchange;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.Planner;
@@ -130,6 +131,7 @@ public class NereidsPlanner extends Planner {
private DescriptorTable descTable;
private FragmentIdMapping<DistributedPlan> distributedPlans;
+ private PlanTranslatorContext planTranslatorContext;
// The cost of optimized plan
private double cost = 0;
private LogicalPlanAdapter logicalPlanAdapter;
@@ -602,7 +604,8 @@ public class NereidsPlanner extends Planner {
return;
}
- PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext(cascadesContext);
+ this.planTranslatorContext = new PlanTranslatorContext(cascadesContext);
+ PlanTranslatorContext planTranslatorContext = this.planTranslatorContext;
PhysicalPlanTranslator physicalPlanTranslator = new PhysicalPlanTranslator(planTranslatorContext,
statementContext.getConnectContext().getStatsErrorEstimator());
SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
@@ -625,7 +628,8 @@ public class NereidsPlanner extends Planner {
scanNodeList.addAll(planTranslatorContext.getScanNodes());
physicalRelations.addAll(planTranslatorContext.getPhysicalRelations());
descTable = planTranslatorContext.getDescTable();
- fragments = new ArrayList<>(planTranslatorContext.getPlanFragments());
+ List<PlanFragment> planFragments = planTranslatorContext.getPlanFragments();
+ fragments = new ArrayList<>(planFragments);
boolean enableQueryCache = sessionVariable.getEnableQueryCache();
for (int seq = 0; seq < fragments.size(); seq++) {
@@ -708,6 +712,18 @@ public class NereidsPlanner extends Planner {
splitFragments(physicalPlan);
doDistribute(canUseNereidsDistributePlanner, explainLevel);
+
+ SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
+ if (sessionVariable.isEnableLocalShufflePlanner() && sessionVariable.isEnableLocalShuffle()) {
+ addLocalExchangeAfterDistribute();
+ }
+ }
+
+ private void addLocalExchangeAfterDistribute() {
+ AddLocalExchange adder = new AddLocalExchange();
+ if (distributedPlans != null && !distributedPlans.isEmpty()) {
+ adder.addLocalExchange(distributedPlans, planTranslatorContext);
+ }
}
protected void doDistribute(boolean canUseNereidsDistributePlanner, ExplainLevel explainLevel) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index b2ca6cfa2b6..ea9eaf8368a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -350,7 +350,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanTranslatorContext context) {
Plan upstream = distribute.child(); // now they're in one fragment but will be split by ExchangeNode.
PlanFragment upstreamFragment = upstream.accept(this, context);
- List<List<Expr>> upstreamDistributeExprs = getDistributeExprs(upstream);
+ List<List<Expr>> upstreamDistributeExprs = getChildrenDistributeExprs(upstream);
DistributionSpec targetDistribution = distribute.getDistributionSpec();
@@ -398,6 +398,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// target data partition
DataPartition targetDataPartition = toDataPartition(targetDistribution, validOutputIds, context);
exchangeNode.setPartitionType(targetDataPartition.getType());
+ exchangeNode.setDistributeExprLists(getDistributeExpr(distribute));
exchangeNode.setChildrenDistributeExprLists(upstreamDistributeExprs);
// its source partition is targetDataPartition. and outputPartition is UNPARTITIONED now, will be set when
// visit its SinkNode
@@ -793,6 +794,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
fileScan.getTableSnapshot().ifPresent(fileQueryScanNode::setQueryTableSnapshot);
fileScan.getScanParams().ifPresent(fileQueryScanNode::setScanParams);
}
+ scanNode.setDistributeExprLists(getDistributeExpr(fileScan));
return getPlanFragmentForPhysicalFileScan(fileScan, context, scanNode);
}
@@ -813,6 +815,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanFragment planFragment = createPlanFragment(emptySetNode,
DataPartition.UNPARTITIONED, emptyRelation);
context.addPlanFragment(planFragment);
+ emptySetNode.setDistributeExprLists(getDistributeExpr(emptyRelation));
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), emptyRelation);
return planFragment;
}
@@ -837,6 +840,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
hudiScanNode.setQueryTableSnapshot(hudiScan.getTableSnapshot().get());
}
hudiScanNode.setSelectedPartitions(hudiScan.getSelectedPartitions());
+ hudiScanNode.setDistributeExprLists(getDistributeExpr(hudiScan));
return getPlanFragmentForPhysicalFileScan(hudiScan, context, hudiScanNode);
}
@@ -858,6 +862,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
DataPartition dataPartition = DataPartition.RANDOM;
PlanFragment planFragment = createPlanFragment(scanNode, dataPartition, fileScan);
context.addPlanFragment(planFragment);
+ scanNode.setDistributeExprLists(getDistributeExpr(fileScan));
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), fileScan);
return planFragment;
}
@@ -896,6 +901,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
context.getScanContext());
olapScanNode.setNereidsId(olapScan.getId());
context.getNereidsIdToPlanNodeIdMap().put(olapScan.getId(), olapScanNode.getId());
+ olapScanNode.setDistributeExprLists(getDistributeExpr(olapScan));
// translate score topn info
if (!olapScan.getScoreOrderKeys().isEmpty()) {
@@ -1021,6 +1027,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanFragment planFragment = createPlanFragment(unionNode, DataPartition.UNPARTITIONED, oneRowRelation);
context.addPlanFragment(planFragment);
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), oneRowRelation);
+
+ unionNode.setDistributeExprLists(getDistributeExpr(oneRowRelation));
return planFragment;
}
@@ -1053,6 +1061,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
context.addScanNode(scanNode, schemaScan);
PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, schemaScan);
context.addPlanFragment(planFragment);
+ scanNode.setDistributeExprLists(getDistributeExpr(schemaScan));
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), schemaScan);
return planFragment;
}
@@ -1069,6 +1078,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, workTableReference);
context.addPlanFragment(planFragment);
+ scanNode.setDistributeExprLists(getDistributeExpr(workTableReference));
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), workTableReference);
return planFragment;
}
@@ -1089,6 +1099,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction();
SessionVariable sv = ConnectContext.get().getSessionVariable();
ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor, sv);
+ scanNode.setDistributeExprLists(getDistributeExpr(tvfRelation));
scanNode.setNereidsId(tvfRelation.getId());
context.getNereidsIdToPlanNodeIdMap().put(tvfRelation.getId(), scanNode.getId());
Utils.execWithUncheckedException(scanNode::init);
@@ -1127,7 +1138,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanTranslatorContext context) {
PlanFragment inputPlanFragment = aggregate.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(aggregate.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(aggregate.child(0));
List<Expression> groupByExpressions = aggregate.getGroupByExpressions();
List<NamedExpression> outputExpressions = aggregate.getOutputExpressions();
@@ -1191,6 +1202,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
AggregationNode aggregationNode = new AggregationNode(context.nextPlanNodeId(),
inputPlanFragment.getPlanRoot(), aggInfo);
+ aggregationNode.setDistributeExprLists(getDistributeExpr(aggregate));
aggregationNode.setChildrenDistributeExprLists(distributeExprLists);
aggregationNode.setNereidsId(aggregate.getId());
@@ -1394,7 +1406,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
public PlanFragment visitPhysicalAssertNumRows(PhysicalAssertNumRows<? extends Plan> assertNumRows,
PlanTranslatorContext context) {
PlanFragment currentFragment = assertNumRows.child().accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(assertNumRows.child());
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(assertNumRows.child());
// we need convert all columns to nullable in AssertNumRows node
// create a tuple for AssertNumRowsNode
@@ -1404,6 +1416,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TupleDescriptor tupleDescriptor = generateTupleDesc(assertNumRows.getOutput(), null, context);
AssertNumRowsNode assertNumRowsNode = new AssertNumRowsNode(context.nextPlanNodeId(),
currentFragment.getPlanRoot(), assertion, tupleDescriptor);
+ assertNumRowsNode.setDistributeExprLists(getDistributeExpr(assertNumRows));
assertNumRowsNode.setChildrenDistributeExprLists(distributeExprLists);
assertNumRowsNode.setNereidsId(assertNumRows.getId());
context.getNereidsIdToPlanNodeIdMap().put(assertNumRows.getId(), assertNumRowsNode.getId());
@@ -1453,6 +1466,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
}
CTEScanNode cteScanNode = new CTEScanNode(context.getScanContext());
+ cteScanNode.setDistributeExprLists(getDistributeExpr(cteConsumer));
translateRuntimeFilter(cteConsumer, cteScanNode, context);
context.getCteScanNodeMap().put(multiCastFragment.getFragmentId(), cteScanNode);
@@ -1564,6 +1578,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.collect(Collectors.toCollection(ArrayList::new));
TableFunctionNode tableFunctionNode = new TableFunctionNode(context.nextPlanNodeId(),
currentFragment.getPlanRoot(), tupleDescriptor.getId(), functionCalls, outputSlotIds, conjuncts);
+ tableFunctionNode.setDistributeExprLists(getDistributeExpr(generate));
tableFunctionNode.setNereidsId(generate.getId());
context.getNereidsIdToPlanNodeIdMap().put(generate.getId(), tableFunctionNode.getId());
addPlanRoot(currentFragment, tableFunctionNode, generate);
@@ -1627,7 +1642,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// NOTICE: We must visit from right to left, to ensure the last fragment is root fragment
PlanFragment rightFragment = hashJoin.child(1).accept(this, context);
PlanFragment leftFragment = hashJoin.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(physicalHashJoin.left(), physicalHashJoin.right());
+ List<List<Expr>> distributeExprLists
+ = getChildrenDistributeExprs(physicalHashJoin.left(), physicalHashJoin.right());
if (JoinUtils.shouldNestedLoopJoin(hashJoin)) {
throw new RuntimeException("Physical hash join could not execute without equal join condition.");
@@ -1678,6 +1694,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
markConjuncts, hashJoin.isMarkJoin());
hashJoinNode.setNereidsId(hashJoin.getId());
context.getNereidsIdToPlanNodeIdMap().put(hashJoin.getId(), hashJoinNode.getId());
+ hashJoinNode.setDistributeExprLists(getDistributeExpr(hashJoin));
hashJoinNode.setChildrenDistributeExprLists(distributeExprLists);
PlanFragment currentFragment = connectJoinNode(hashJoinNode, leftFragment, rightFragment, context, hashJoin);
@@ -1897,7 +1914,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// PhysicalPlan plan, PlanVisitor visitor, Context context).
PlanFragment rightFragment = nestedLoopJoin.child(1).accept(this, context);
PlanFragment leftFragment = nestedLoopJoin.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(nestedLoopJoin.child(0), nestedLoopJoin.child(1));
+ List<List<Expr>> distributeExprLists
+ = getChildrenDistributeExprs(nestedLoopJoin.child(0), nestedLoopJoin.child(1));
PlanNode leftFragmentPlanRoot = leftFragment.getPlanRoot();
PlanNode rightFragmentPlanRoot = rightFragment.getPlanRoot();
@@ -1918,6 +1936,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
nestedLoopJoin.isMarkJoin());
nestedLoopJoinNode.setNereidsId(nestedLoopJoin.getId());
context.getNereidsIdToPlanNodeIdMap().put(nestedLoopJoin.getId(), nestedLoopJoinNode.getId());
+ nestedLoopJoinNode.setDistributeExprLists(getDistributeExpr(nestedLoopJoin));
nestedLoopJoinNode.setChildrenDistributeExprLists(distributeExprLists);
if (nestedLoopJoin.getStats() != null) {
nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount());
@@ -2116,9 +2135,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
public PlanFragment visitPhysicalPartitionTopN(PhysicalPartitionTopN<? extends Plan> partitionTopN,
PlanTranslatorContext context) {
PlanFragment inputFragment = partitionTopN.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(partitionTopN.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(partitionTopN.child(0));
PartitionSortNode partitionSortNode = translatePartitionSortNode(
partitionTopN, inputFragment.getPlanRoot(), context);
+ partitionSortNode.setDistributeExprLists(getDistributeExpr(partitionTopN));
partitionSortNode.setChildrenDistributeExprLists(distributeExprLists);
addPlanRoot(inputFragment, partitionSortNode, partitionTopN);
// in pipeline engine, we use parallel scan by default, but it broke the rule of data distribution
@@ -2297,12 +2317,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
for (Plan plan : recursiveCte.children()) {
childrenFragments.add(plan.accept(this, context));
}
- List<List<Expr>> distributeExprLists = getDistributeExprs(recursiveCte.children().toArray(new Plan[0]));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(recursiveCte.children().toArray(new Plan[0]));
TupleDescriptor setTuple = generateTupleDesc(recursiveCte.getOutput(), null, context);
RecursiveCteNode recursiveCteNode = new RecursiveCteNode(context.nextPlanNodeId(), setTuple.getId(),
recursiveCte.getCteName(), recursiveCte.isUnionAll());
+ recursiveCteNode.setDistributeExprLists(getDistributeExpr(recursiveCte));
recursiveCteNode.setChildrenDistributeExprLists(distributeExprLists);
recursiveCteNode.setNereidsId(recursiveCte.getId());
context.getNereidsIdToPlanNodeIdMap().put(recursiveCte.getId(), recursiveCteNode.getId());
@@ -2370,7 +2391,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
for (Plan plan : setOperation.children()) {
childrenFragments.add(plan.accept(this, context));
}
- List<List<Expr>> distributeExprLists = getDistributeExprs(setOperation.children().toArray(new Plan[0]));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(setOperation.children().toArray(new Plan[0]));
TupleDescriptor setTuple = generateTupleDesc(setOperation.getOutput(), null, context);
SetOperationNode setOperationNode;
@@ -2384,6 +2405,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
} else {
throw new RuntimeException("not support set operation type " + setOperation);
}
+ setOperationNode.setDistributeExprLists(getDistributeExpr(setOperation));
setOperationNode.setChildrenDistributeExprLists(distributeExprLists);
setOperationNode.setNereidsId(setOperation.getId());
context.getNereidsIdToPlanNodeIdMap().put(setOperation.getId(), setOperationNode.getId());
@@ -2493,12 +2515,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
public PlanFragment visitPhysicalQuickSort(PhysicalQuickSort<? extends Plan> sort,
PlanTranslatorContext context) {
PlanFragment inputFragment = sort.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(sort.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(sort.child(0));
// 2. According to the type of sort, generate physical plan
if (!sort.getSortPhase().isMerge()) {
// For localSort or Gather->Sort, we just need to add sortNode
SortNode sortNode = translateSortNode(sort, inputFragment.getPlanRoot(), context);
+ sortNode.setDistributeExprLists(getDistributeExpr(sort));
sortNode.setChildrenDistributeExprLists(distributeExprLists);
addPlanRoot(inputFragment, sortNode, sort);
} else {
@@ -2515,6 +2538,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
inputFragment.getChild(0).getSink().setMerge(true);
}
sortNode.setMergeByExchange();
+ sortNode.setDistributeExprLists(getDistributeExpr(sort));
sortNode.setChildrenDistributeExprLists(distributeExprLists);
}
return inputFragment;
@@ -2523,7 +2547,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, PlanTranslatorContext context) {
PlanFragment inputFragment = topN.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(topN.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(topN.child(0));
// 2. According to the type of sort, generate physical plan
if (!topN.getSortPhase().isMerge()) {
// For localSort or Gather->Sort, we just need to add TopNNode
@@ -2559,6 +2583,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
}
}
+ sortNode.setDistributeExprLists(
+ CollectionUtils.isEmpty(distributeExprLists) ? null : distributeExprLists.get(0)
+ );
sortNode.setChildrenDistributeExprLists(distributeExprLists);
addPlanRoot(inputFragment, sortNode, topN);
} else {
@@ -2572,6 +2599,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return inputFragment;
}
ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot();
+ exchangeNode.setDistributeExprLists(
+ CollectionUtils.isEmpty(distributeExprLists) ? null : distributeExprLists.get(0)
+ );
exchangeNode.setChildrenDistributeExprLists(distributeExprLists);
exchangeNode.setMergeInfo(((SortNode) exchangeNode.getChild(0)).getSortInfo());
if (inputFragment.hasChild(0) && inputFragment.getChild(0).getSink() != null) {
@@ -2588,7 +2618,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalRepeat(PhysicalRepeat<? extends Plan> repeat, PlanTranslatorContext context) {
PlanFragment inputPlanFragment = repeat.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(repeat.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(repeat.child(0));
List<Expression> flattenGroupingExpressions = repeat.getGroupByExpressions();
Set<Slot> preRepeatExpressions = Sets.newLinkedHashSet();
@@ -2639,6 +2669,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
allSlotId, repeat.computeGroupingFunctionsValues());
repeatNode.setNereidsId(repeat.getId());
context.getNereidsIdToPlanNodeIdMap().put(repeat.getId(), repeatNode.getId());
+ repeatNode.setDistributeExprLists(getDistributeExpr(repeat));
repeatNode.setChildrenDistributeExprLists(distributeExprLists);
addPlanRoot(inputPlanFragment, repeatNode, repeat);
updateLegacyPlanIdToPhysicalPlan(inputPlanFragment.getPlanRoot(), repeat);
@@ -2649,7 +2680,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
public PlanFragment visitPhysicalWindow(PhysicalWindow<? extends Plan> physicalWindow,
PlanTranslatorContext context) {
PlanFragment inputPlanFragment = physicalWindow.child(0).accept(this, context);
- List<List<Expr>> distributeExprLists = getDistributeExprs(physicalWindow.child(0));
+ List<List<Expr>> distributeExprLists = getChildrenDistributeExprs(physicalWindow.child(0));
// 1. translate to old optimizer variable
// variable in Nereids
@@ -2714,6 +2745,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
);
analyticEvalNode.setNereidsId(physicalWindow.getId());
context.getNereidsIdToPlanNodeIdMap().put(physicalWindow.getId(), analyticEvalNode.getId());
+ analyticEvalNode.setDistributeExprLists(getDistributeExpr(physicalWindow));
analyticEvalNode.setChildrenDistributeExprLists(distributeExprLists);
PlanNode root = inputPlanFragment.getPlanRoot();
if (root instanceof SortNode) {
@@ -2751,6 +2783,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
materializeNode.setLazyColumns(materialize.getLazyColumns());
materializeNode.setLocations(materialize.getLazySlotLocations());
materializeNode.setColumnIdxsLists(materialize.getLazyBaseColumnIndices());
+ materializeNode.setDistributeExprLists(getDistributeExpr(materialize));
List<Boolean> rowStoreFlags = new ArrayList<>();
for (Relation relation : materialize.getRelations()) {
@@ -2790,6 +2823,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction();
SessionVariable sv = ConnectContext.get().getSessionVariable();
ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor, sv);
+ scanNode.setDistributeExprLists(getDistributeExpr(tvfRelation));
scanNode.setNereidsId(tvfRelation.getId());
context.getNereidsIdToPlanNodeIdMap().put(tvfRelation.getId(), scanNode.getId());
Utils.execWithUncheckedException(scanNode::init);
@@ -3280,7 +3314,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return false;
}
- private List<List<Expr>> getDistributeExprs(Plan ... children) {
+ private List<List<Expr>> getChildrenDistributeExprs(Plan ... children) {
List<List<Expr>> distributeExprLists = Lists.newArrayList();
for (Plan child : children) {
DistributionSpec spec = ((PhysicalPlan) child).getPhysicalProperties().getDistributionSpec();
@@ -3289,6 +3323,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return distributeExprLists;
}
+ private List<Expr> getDistributeExpr(PhysicalPlan physicalPlan) {
+ return getDistributeExpr(
+ physicalPlan.getOutputExprIds(),
+ physicalPlan.getPhysicalProperties().getDistributionSpec()
+ );
+ }
+
private List<Expr> getDistributeExpr(List<ExprId> childOutputIds, DistributionSpec spec) {
if (spec instanceof DistributionSpecHash) {
DistributionSpecHash distributionSpecHash = (DistributionSpecHash) spec;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 4118399b91d..db9c8964cec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -109,10 +109,20 @@ public class PlanTranslatorContext {
private final Map<ScanNode, Set<SlotId>> statsUnknownColumnsMap = Maps.newHashMap();
+ private final Map<PlanNodeId, Boolean> serialAncestorInPipelineMap = Maps.newHashMap();
+
+ // Whether the current fragment uses LocalShuffleAssignedJob (pooling scan with
+ // ignoreDataDistribution → _parallel_instances=1 in BE). When true, serial operators
+ // indicate real pipeline bottlenecks needing PASSTHROUGH fan-out (heavy_ops).
private boolean isTopMaterializeNode = true;
private final Set<SlotId> virtualColumnIds = Sets.newHashSet();
+ // Used by AddLocalExchange: tracks whether any serial operator exists
+ // between the current node and the pipeline's sink (downstream serial check).
+ // Pipeline-splitting nodes reset this to false; serial non-splitting nodes set it to true.
+ private boolean serialAncestorInCurrentPipeline;
+
/** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx) {
this.connectContext = ctx.getConnectContext();
@@ -234,6 +244,14 @@ public class PlanTranslatorContext {
return nodeIdGenerator.getNextId();
}
+ public void setHasSerialAncestorInPipeline(PlanNode node, boolean hasSerialAncestorInPipeline) {
+ serialAncestorInPipelineMap.put(node.getId(), hasSerialAncestorInPipeline);
+ }
+
+ public boolean hasSerialAncestorInPipeline(PlanNode node) {
+ return serialAncestorInPipelineMap.getOrDefault(node.getId(), false);
+ }
+
public SlotDescriptor addSlotDesc(TupleDescriptor t) {
return descTable.addSlotDescriptor(t);
}
@@ -373,4 +391,12 @@ public class PlanTranslatorContext {
public Set<SlotId> getVirtualColumnIds() {
return virtualColumnIds;
}
+
+ public boolean hasSerialAncestorInCurrentPipeline() {
+ return serialAncestorInCurrentPipeline;
+ }
+
+ public void setSerialAncestorInCurrentPipeline(boolean serialAncestorInCurrentPipeline) {
+ this.serialAncestorInCurrentPipeline = serialAncestorInCurrentPipeline;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
index fffc71ace23..8fa31c492f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
@@ -31,6 +31,18 @@ public class AccessPathInfo {
// (not the char data) is needed — agreed with BE as the special path component name.
public static final String ACCESS_STRING_OFFSET = "OFFSET";
+ // s: map<int, string>
+ // map
+ // key: int true
+ // value: string true
+ // s.*
+ //
+ // s: string
+ // len: int always true
+ // body: null true
+ //
+ // select map_keys(s), map_valeus(s), length(s), substring(s,
+
private DataType prunedType;
// allAccessPaths is used to record all access path include predicate access path and non-predicate access path,
// and predicateAccessPaths only contains the predicate access path.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 09acde2ad6f..2f5dcbb7cfc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -144,6 +144,7 @@ public enum PlanType {
PHYSICAL_RECURSIVE_CTE,
PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD,
PHYSICAL_DISTRIBUTE,
+ PHYSICAL_LOCAL_DISTRIBUTE,
PHYSICAL_EXCEPT,
PHYSICAL_FILTER,
PHYSICAL_GENERATE,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
index 0069ddd37db..f50a32b2913 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
@@ -257,7 +257,8 @@ public class DistributePlanner {
AssignedJob[] instances = new AssignedJob[bucketNum];
for (AssignedJob instanceJob : unsorted) {
BucketScanSource bucketScanSource = (BucketScanSource) instanceJob.getScanSource();
- for (Integer bucketIndex : bucketScanSource.bucketIndexToScanNodeToTablets.keySet()) {
+ Iterable<Integer> bucketIndices = bucketScanSource.bucketIndexToScanNodeToTablets.keySet();
+ for (Integer bucketIndex : bucketIndices) {
if (instances[bucketIndex] != null) {
throw new IllegalStateException(
"Multi instances scan same buckets: " + instances[bucketIndex] + " and " + instanceJob
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
index 6a2315966bc..b6319da5b9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
@@ -270,7 +270,7 @@ public class UnassignedScanBucketOlapTableJob extends AbstractUnassignedScanJob
Map<Integer, Map<ScanNode, ScanRanges>> serialScanRanges = Maps.newLinkedHashMap();
Map<Integer, Map<ScanNode, ScanRanges>> nonSerialScanRanges = Maps.newLinkedHashMap();
for (ScanNode scanNode : scanNodes) {
- if (scanNode.isSerialOperator()) {
+ if (scanNode.isSerialNode()) {
collectScanRanges(totalScanSource, scanNode, serialScanRanges);
} else {
collectScanRanges(totalScanSource, scanNode, nonSerialScanRanges);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AddLocalExchange.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AddLocalExchange.java
new file mode 100644
index 00000000000..8b67d8d7734
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AddLocalExchange.java
@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
+import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
+import org.apache.doris.planner.LocalExchangeNode.RequireHash;
+
+/** AddLocalExchange */
+public class AddLocalExchange {
+ /** addLocalExchange with distributed plans, skipping single-instance fragments.
+ * BE's _plan_local_exchange checks _num_instances which is the per-BE instance count.
+ * With _num_instances<=1 all pipelines on that BE have 1 task so local exchange is a no-op.
+ * We must use the same per-BE semantics: skip when every BE has at most 1 instance.
+ * Using global instanceCount would insert LE for fragments where 2 BEs each have 1 instance
+ * (global=2, per-BE=1), causing pipeline task mismatch and deadlock. */
+ public void addLocalExchange(FragmentIdMapping<DistributedPlan> distributedPlans,
+ PlanTranslatorContext context) {
+ String qid = context.getConnectContext() != null
+ ? org.apache.doris.common.util.DebugUtil.printId(context.getConnectContext().queryId()) : "?";
+ for (DistributedPlan plan : distributedPlans.values()) {
+ PipelineDistributedPlan pipePlan = (PipelineDistributedPlan) plan;
+ long maxPerBeInstances = pipePlan.getInstanceJobs().stream()
+ .collect(java.util.stream.Collectors.groupingBy(
+ j -> j.getAssignedWorker().id(), java.util.stream.Collectors.counting()))
+ .values().stream().mapToLong(Long::longValue).max().orElse(0);
+ PlanFragment fragment = pipePlan.getFragmentJob().getFragment();
+ boolean isPooling = pipePlan.getInstanceJobs().stream().anyMatch(
+ org.apache.doris.nereids.trees.plans.distribute.worker.job
+ .LocalShuffleAssignedJob.class::isInstance);
+ if (maxPerBeInstances <= 1) {
+ continue;
+ }
+ String rootBefore = fragment.getPlanRoot().getClass().getSimpleName()
+ + "#" + fragment.getPlanRoot().getId();
+ addLocalExchangeForFragment(fragment, context);
+ String rootAfter = fragment.getPlanRoot().getClass().getSimpleName()
+ + "#" + fragment.getPlanRoot().getId();
+ org.apache.logging.log4j.LogManager.getLogger(AddLocalExchange.class).info(
+ "addLocalExchange: qid={} fragment={} maxPerBe={} pooling={} root={}→{} changed={}",
+ qid, fragment.getFragmentId(), maxPerBeInstances, isPooling,
+ rootBefore, rootAfter, !rootBefore.equals(rootAfter));
+ }
+ }
+
+ private void addLocalExchangeForFragment(PlanFragment fragment, PlanTranslatorContext context) {
+ DataSink sink = fragment.getSink();
+ LocalExchangeTypeRequire require = sink == null
+ ? LocalExchangeTypeRequire.noRequire() : sink.getLocalExchangeTypeRequire();
+ PlanNode root = fragment.getPlanRoot();
+ context.setHasSerialAncestorInPipeline(root, false);
+ Pair<PlanNode, LocalExchangeType> output = root
+ .enforceAndDeriveLocalExchange(context, null, require);
+ PlanNode newRoot = output.first;
+ // The fragment data sink (DataStreamSink, OlapTableSink) runs in the same pipeline
+ // as the root. If the root will be serial on BE, the sink pipeline has 1 task —
+ // only instance 0 sends data, others hang or miss writes.
+ // Insert PASSTHROUGH fan-out so sink runs with _num_instances tasks.
+ // This matches BE-native's default required_data_distribution():
+ // _child->is_serial_operator() ? PASSTHROUGH : NOOP
+ if (newRoot.isSerialOperatorOnBe(context.getConnectContext())) {
+ newRoot = new LocalExchangeNode(context.nextPlanNodeId(), newRoot,
+ LocalExchangeType.PASSTHROUGH, null);
+ }
+ if (newRoot != root) {
+ fragment.setPlanRoot(newRoot);
+ }
+ validateNoSerialWithoutLocalExchange(fragment.getPlanRoot(), context.getConnectContext());
+ }
+
+ /**
+ * In a local-shuffle fragment, the root check above guarantees the root pipeline
+ * has N tasks. Any serial operator reduces its pipeline to 1 task. If this serial
+ * operator feeds into a non-serial parent without LocalExchangeNode in between,
+ * some pipelines have 1 task while others have N → shared_state mismatch, data loss.
+ *
+ * Serial→serial chains are fine (all at 1 task, consistent). Only the transition
+ * from serial to non-serial needs LE to restore parallelism.
+ */
+ private void validateNoSerialWithoutLocalExchange(PlanNode node,
+ org.apache.doris.qe.ConnectContext context) {
+ for (PlanNode child : node.getChildren()) {
+ validateNoSerialWithoutLocalExchange(child, context);
+ if (child.isSerialOperatorOnBe(context)
+ && !(child instanceof LocalExchangeNode)
+ && !(node instanceof LocalExchangeNode)
+ && !(node instanceof ExchangeNode)
+ && !node.isSerialOperatorOnBe(context)) {
+ org.apache.logging.log4j.LogManager.getLogger(AddLocalExchange.class).warn(
+ "Serial " + child.getClass().getSimpleName() + "(id=" + child.getId()
+ + ") feeds into non-serial " + node.getClass().getSimpleName()
+ + "(id=" + node.getId() + ") without LocalExchangeNode"
+ + " in fragment " + node.getFragment().getFragmentId()
+ + ". FE should insert LocalExchangeNode to restore parallelism.");
+ }
+ }
+ }
+
+ public static boolean isColocated(PlanNode plan) {
+ if (plan instanceof AggregationNode) {
+ return ((AggregationNode) plan).isColocate() && isColocated(plan.getChild(0));
+ } else if (plan instanceof OlapScanNode) {
+ return true;
+ } else if (plan instanceof SelectNode) {
+ return isColocated(plan.getChild(0));
+ } else if (plan instanceof HashJoinNode) {
+ return ((HashJoinNode) plan).isColocate()
+ && (isColocated(plan.getChild(0)) || isColocated(plan.getChild(1)));
+ } else if (plan instanceof SetOperationNode) {
+ if (!((SetOperationNode) plan).isColocate()) {
+ return false;
+ }
+ for (PlanNode child : plan.getChildren()) {
+ if (isColocated(child)) {
+ return true;
+ }
+ }
+ return false;
+ } else {
+ return false;
+ }
+ }
+
+ public static LocalExchangeType resolveExchangeType(LocalExchangeTypeRequire require,
+ PlanTranslatorContext translatorContext, PlanNode parent, PlanNode child) {
+ // Only generic RequireHash adapts to LOCAL_EXECUTION_HASH_SHUFFLE.
+ // Explicit RequireSpecific (GLOBAL_EXECUTION_HASH_SHUFFLE, BUCKET_HASH_SHUFFLE, etc.)
+ // must never be degraded — if they appear in an invalid context, the plan is wrong.
+ //
+ // Always prefer LOCAL_EXECUTION_HASH_SHUFFLE for FE-planned intra-fragment hash exchanges.
+ // GLOBAL_EXECUTION_HASH_SHUFFLE requires shuffle_idx_to_instance_idx which may be empty
+ // for fragments with non-hash sinks (UNPARTITIONED/MERGE). LOCAL_HASH is always safe
+ // since it partitions by local instance count without needing external shuffle maps.
+ if (require instanceof RequireHash) {
+ return LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE;
+ }
+ return require.preferType();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index 0f9d34ae826..562199a2e89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -27,7 +27,13 @@ import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.SortInfo;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.planner.normalize.Normalizer;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TAggregationNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
@@ -239,7 +245,7 @@ public class AggregationNode extends PlanNode {
// If `GroupingExprs` is empty and agg need to finalize, the result must be output by single instance
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return aggInfo.getGroupingExprs().isEmpty() && needsFinalize;
}
@@ -247,6 +253,10 @@ public class AggregationNode extends PlanNode {
isColocate = colocate;
}
+ public boolean isColocate() {
+ return isColocate;
+ }
+
public void setSortByGroupKey(SortInfo sortByGroupKey) {
this.sortByGroupKey = sortByGroupKey;
}
@@ -258,4 +268,97 @@ public class AggregationNode extends PlanNode {
public void setQueryCacheCandidate(boolean queryCacheCandidate) {
this.queryCacheCandidate = queryCacheCandidate;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+
+ ConnectContext connectContext = translatorContext.getConnectContext();
+ SessionVariable sessionVariable = connectContext.getSessionVariable();
+
+ LocalExchangeTypeRequire requireChild;
+ if (canUseDistinctStreamingAgg(sessionVariable)) {
+ // DistinctStreamingAggOperatorX in BE
+ if (needsFinalize || (!aggInfo.getGroupingExprs().isEmpty() && !useStreamingPreagg)) {
+ if (AddLocalExchange.isColocated(this)) {
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ } else {
+ requireChild = parentRequire.autoRequireHash();
+ }
+ } else if (sessionVariable.enableDistinctStreamingAggForcePassthrough) {
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else if (children.get(0).isSerialOperatorOnBe(connectContext)) {
+ // BE base class: _child->is_serial_operator() ? PASSTHROUGH : NOOP
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ }
+ } else if (useStreamingPreagg) {
+ // StreamingAggOperatorX in BE: base class required_data_distribution():
+ // _child->is_serial_operator() ? PASSTHROUGH : NOOP
+ // Special: directly above HashJoin probe with force-passthrough → PASSTHROUGH.
+ if (children.get(0) instanceof HashJoinNode
+ && sessionVariable.enableStreamingAggHashJoinForcePassthrough) {
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else if (children.get(0).isSerialOperatorOnBe(connectContext)) {
+ // BE: _child->is_serial_operator() ? PASSTHROUGH : NOOP
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ }
+ } else {
+ // AggSinkOperatorX in BE: required_data_distribution() override
+ if (aggInfo.getGroupingExprs().isEmpty()) {
+ if (needsFinalize) {
+ // Finalize agg, no group key: NOOP (single serial instance collects all)
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ } else {
+ // Serialize agg, no group key: BE → _child->is_serial_operator() ? PT : NOOP
+ if (children.get(0).isSerialOperatorOnBe(connectContext)) {
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ } else {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ }
+ }
+ } else if (AddLocalExchange.isColocated(this)) {
+ // Colocate: BUCKET_HASH_SHUFFLE
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ } else {
+ // Non-colocate, has group key: BE → GLOBAL_HASH
+ // FE uses parentRequire as proxy (no full need_to_local_exchange equivalent)
+ if (!needsFinalize
+ && sessionVariable.getShuffledAggNodeIds().contains(this.getId().asInt())) {
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ } else if (hasPartitionExprs(parentRequire)) {
+ requireChild = parentRequire.autoRequireHash();
+ } else {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ }
+ }
+ }
+
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, requireChild);
+ children = Lists.newArrayList(enforceResult.first);
+ return Pair.of(this, enforceResult.second);
+ }
+
+ @Override
+ protected List<Expr> getSemanticPartitionExprs() {
+ return aggInfo.getGroupingExprs();
+ }
+
+ private boolean canUseDistinctStreamingAgg(SessionVariable sessionVariable) {
+ return aggInfo.getAggregateExprs().isEmpty() && sortByGroupKey == null
+ && sessionVariable.enableDistinctStreamingAggregation;
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ // Non-streaming AGG is a pipeline breaker: child is in AGG_Sink pipeline,
+ // parent is in AGG_Source pipeline. Reset inherited serial flag from parent
+ // (different pipeline), but enforceRequire still adds this node's own
+ // isSerialNode() so the child sees AGG_Sink's serial status correctly.
+ return !useStreamingPreagg;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
index 2ede7ac4e52..75d20d1fa4e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
@@ -27,6 +27,10 @@ import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.OrderByElement;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TAnalyticNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@@ -176,7 +180,73 @@ public class AnalyticEvalNode extends PlanNode {
* all data should be input in this node to ensure the global ordering by colB.
*/
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return partitionExprs.isEmpty();
}
+
+ @Override
+ protected List<Expr> getSemanticPartitionExprs() {
+ return partitionExprs;
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ LocalExchangeTypeRequire requireChild;
+ LocalExchangeType outputType = null;
+ if (partitionExprs.isEmpty()) {
+ // Serial AnalyticEval (OVER() with no PARTITION BY):
+ // Must NOT have any LocalExchange between AnalyticEval and its child.
+ // On BE, AnalyticSink and AnalyticSource share state (source_deps/sink_deps).
+ // A LocalExchange below would restore the AnalyticSink pipeline to _num_instances
+ // tasks while the serial AnalyticSource pipeline stays at 1 task.
+ //
+ // Use enforceRequire with noRequire to traverse children, then strip any
+ // LocalExchange the child inserted (e.g., Exchange wrapping itself with PASSTHROUGH).
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, LocalExchangeTypeRequire.noRequire());
+ PlanNode newChild = enforceResult.first;
+ if (newChild instanceof LocalExchangeNode) {
+ newChild = newChild.getChild(0);
+ }
+ children = Lists.newArrayList(newChild);
+ // Return NOOP: the serial AnalyticSource pipeline has 1 task, we don't provide
+ // fan-out ourselves. The parent's enforceRequire framework-level serial check
+ // will see our serial status and insert PASSTHROUGH LE above us if needed.
+ return Pair.of(this, LocalExchangeType.NOOP);
+ } else if (orderByElements.isEmpty()) {
+ if (AddLocalExchange.isColocated(this)) {
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ outputType = AddLocalExchange.resolveExchangeType(
+ LocalExchangeTypeRequire.requireHash(), translatorContext, this,
+ children.get(0));
+ } else {
+ // Non-colocated analytic with PARTITION BY but no ORDER BY:
+ // The parent SortNode (mergeByExchange) will insert PASSTHROUGH above us,
+ // which is what BE does natively. Don't force a hash exchange here.
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ }
+ } else if (children.get(0).isSerialOperatorOnBe(translatorContext.getConnectContext())) {
+ // BE base class: _child->is_serial_operator() ? PASSTHROUGH : NOOP
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ outputType = LocalExchangeType.PASSTHROUGH;
+ } else {
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ }
+
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, requireChild);
+ children = Lists.newArrayList(enforceResult.first);
+ if (outputType == null) {
+ outputType = enforceResult.second;
+ }
+ return Pair.of(this, outputType);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
index dcf683f0783..5784d2050a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
@@ -19,11 +19,17 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.AssertNumRowsElement;
import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TAssertNumRowsNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
+import com.google.common.collect.Lists;
+
/**
* Assert num rows node is used to determine whether the number of rows is less than desired num of rows.
* The rows are the result of subqueryString.
@@ -78,7 +84,17 @@ public class AssertNumRowsNode extends PlanNode {
}
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return true;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+
+ Pair<PlanNode, LocalExchangeType> enforceResult = enforceRequire(
+ translatorContext, children.get(0), 0, LocalExchangeTypeRequire.requirePassthrough());
+ children = Lists.newArrayList(enforceResult.first);
+ return Pair.of(this, LocalExchangeType.PASSTHROUGH);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/CTEScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/CTEScanNode.java
index 13ac382bf16..92bf1974b39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/CTEScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/CTEScanNode.java
@@ -19,7 +19,11 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRangeLocations;
@@ -55,4 +59,10 @@ public class CTEScanNode extends ScanNode {
// NO real action to be taken, just a wrapper
return null;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
index 07da3d4f9ba..f8c27eae5c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
@@ -20,6 +20,7 @@
package org.apache.doris.planner;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TExplainLevel;
@@ -65,4 +66,8 @@ public abstract class DataSink {
public void setMerge(boolean merge) {
isMerge = merge;
}
+
+ public LocalExchangeTypeRequire getLocalExchangeTypeRequire() {
+ return LocalExchangeTypeRequire.noRequire();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
index 00d84cf0fbb..e6998ee72ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
@@ -239,4 +239,5 @@ public class DataStreamSink extends DataSink {
result.setStreamSink(tStreamSink);
return result;
}
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
index d89bfc0b4ee..3a731d28a90 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
@@ -18,6 +18,10 @@
package org.apache.doris.planner;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -48,4 +52,10 @@ public class EmptySetNode extends PlanNode {
public int getNumInstances() {
return 1;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
index 7274f731c5a..76315210e5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
@@ -23,6 +23,10 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.SortInfo;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExchangeNode;
import org.apache.doris.thrift.TExplainLevel;
@@ -98,10 +102,7 @@ public class ExchangeNode extends PlanNode {
@Override
protected void toThrift(TPlanNode msg) {
- // If this fragment has another scan node, this exchange node is serial or not should be decided by the scan
- // node.
- msg.setIsSerialOperator((isSerialOperator() || fragment.hasSerialScanNode())
- && fragment.useSerialSource(ConnectContext.get()));
+ msg.setIsSerialOperator(isSerialOperatorOnBe(ConnectContext.get()));
msg.node_type = TPlanNodeType.EXCHANGE_NODE;
msg.exchange_node = new TExchangeNode();
for (TupleId tid : tupleIds) {
@@ -152,18 +153,38 @@ public class ExchangeNode extends PlanNode {
* So FRAGMENT 0 should not use serial source.
*/
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isUseSerialExchange()
|| partitionType == TPartitionType.UNPARTITIONED) && mergeInfo == null;
}
+ @Override
+ public boolean isSerialOperatorOnBe(ConnectContext context) {
+ return fragment != null
+ && (isSerialNode() || fragment.hasSerialScanNode())
+ && fragment.useSerialSource(context);
+ }
+
@Override
public boolean hasSerialChildren() {
- return isSerialOperator();
+ return isSerialNode();
}
@Override
public boolean hasSerialScanChildren() {
return false;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ // Report actual distribution. Serial handling is done by the framework
+ // (enforceRequire step 2.5 overrides serial child output to NOOP).
+ if (partitionType == TPartitionType.HASH_PARTITIONED) {
+ return Pair.of(this, LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE);
+ } else if (partitionType == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED) {
+ return Pair.of(this, LocalExchangeType.BUCKET_HASH_SHUFFLE);
+ }
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index a98422bb2e0..87c3094a361 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -28,7 +28,11 @@ import org.apache.doris.analysis.JoinOperator;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TEqJoinCondition;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.THashJoinNode;
@@ -124,6 +128,10 @@ public class HashJoinNode extends JoinNodeBase {
colocateReason = reason;
}
+ public boolean isColocate() {
+ return isColocate;
+ }
+
public Map<ExprId, SlotId> getHashOutputExprSlotIdMap() {
return hashOutputExprSlotIdMap;
}
@@ -285,4 +293,71 @@ public class HashJoinNode extends JoinNodeBase {
public List<Expr> getMarkJoinConjuncts() {
return markJoinConjuncts;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+
+ LocalExchangeTypeRequire probeSideRequire;
+ LocalExchangeTypeRequire buildSideRequire;
+ LocalExchangeType outputType = null;
+
+ if (joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN) {
+ buildSideRequire = probeSideRequire = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ } else if (distrMode == DistributionMode.BROADCAST) {
+ // BE: _child->is_serial_operator() ? PASSTHROUGH/PASS_TO_ONE : NOOP
+ boolean probeChildSerial = children.get(0).isSerialOperatorOnBe(
+ translatorContext.getConnectContext());
+ boolean buildChildSerial = children.get(1).isSerialOperatorOnBe(
+ translatorContext.getConnectContext());
+ probeSideRequire = probeChildSerial
+ ? LocalExchangeTypeRequire.requirePassthrough()
+ : LocalExchangeTypeRequire.noRequire();
+ buildSideRequire = buildChildSerial
+ ? LocalExchangeTypeRequire.requirePassToOne()
+ : LocalExchangeTypeRequire.noRequire();
+ // For serial probe: output is PASSTHROUGH (data from single instance).
+ // For non-serial probe: propagate probe side's actual distribution.
+ outputType = probeChildSerial ? LocalExchangeType.PASSTHROUGH : null;
+ } else if (isColocate() || isBucketShuffle()) {
+ probeSideRequire = LocalExchangeTypeRequire.requireBucketHash();
+ // For BUCKET_SHUFFLE with serial build child: use requireBucketHash() (not
+ // requirePassToOne()). Unlike BROADCAST joins, BUCKET_SHUFFLE has no shared
+ // hash table mechanism — PASS_TO_ONE routes all data to task 0 while tasks 1..N-1
+ // build empty hash tables, losing rows. BUCKET_HASH_SHUFFLE correctly distributes
+ // build data by bucket to match the probe side's bucket distribution.
+ // The serial exchange returns NOOP, so enforceRequire() will insert a
+ // BUCKET_HASH_SHUFFLE local exchange (with PASSTHROUGH fan-out for heavy-ops
+ // bottleneck avoidance).
+ buildSideRequire = LocalExchangeTypeRequire.requireBucketHash();
+ outputType = AddLocalExchange.resolveExchangeType(
+ LocalExchangeTypeRequire.requireBucketHash(), translatorContext, this,
+ children.get(0));
+ } else {
+ // Use requireHash() (not requireGlobalExecutionHash()) so that resolveExchangeType()
+ // can downgrade to LOCAL_EXECUTION_HASH_SHUFFLE via shouldUseLocalExecutionHash().
+ // This matches BE-native behavior where use_serial_exchange=true sets _use_serial_source=true,
+ // causing _add_local_exchange_impl to use LOCAL (not GLOBAL) hash shuffle.
+ // With use_serial_exchange=false, the upstream ExchangeNode already outputs
+ // GLOBAL_EXECUTION_HASH_SHUFFLE which satisfies requireHash() — no new exchange inserted.
+ buildSideRequire = probeSideRequire = LocalExchangeTypeRequire.requireHash();
+ outputType = null; // derived from probeResult.second below
+ }
+
+ Pair<PlanNode, LocalExchangeType> probeResult = enforceRequire(
+ translatorContext, children.get(0), 0, probeSideRequire);
+ Pair<PlanNode, LocalExchangeType> buildResult = enforceRequire(
+ translatorContext, children.get(1), 1, buildSideRequire);
+ this.children = Lists.newArrayList(probeResult.first, buildResult.first);
+ if (outputType == null) {
+ outputType = probeResult.second;
+ }
+ return Pair.of(this, outputType);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return childIndex == 1;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java
new file mode 100644
index 00000000000..3674f0f6db8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java
@@ -0,0 +1,316 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ExchangeNode.java
+// and modified by Doris
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.ExprToThriftVisitor;
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TExpr;
+import org.apache.doris.thrift.TLocalExchangeNode;
+import org.apache.doris.thrift.TLocalPartitionType;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/** LocalExchangeNode */
+public class LocalExchangeNode extends PlanNode {
+ public static final String EXCHANGE_NODE = "LOCAL-EXCHANGE";
+
+ private LocalExchangeType exchangeType;
+
+ /**
+ * use for Nereids only.
+ */
+ public LocalExchangeNode(PlanNodeId id, PlanNode inputNode, LocalExchangeType exchangeType) {
+ this(id, inputNode, exchangeType, null);
+ }
+
+ public LocalExchangeNode(PlanNodeId id, PlanNode inputNode, LocalExchangeType exchangeType,
+ List<Expr> distributeExprs) {
+ super(id, inputNode, EXCHANGE_NODE);
+ this.offset = 0;
+ this.limit = -1;
+ this.conjuncts = Collections.emptyList();
+ this.children.add(inputNode);
+ this.exchangeType = exchangeType;
+ this.fragment = inputNode.getFragment();
+
+ List<Expr> hashExprs = distributeExprs;
+ boolean isHashShuffle = (exchangeType == LocalExchangeType.BUCKET_HASH_SHUFFLE
+ || exchangeType == LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE
+ || exchangeType == LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE);
+ if (isHashShuffle && hashExprs != null && !hashExprs.isEmpty()) {
+ setDistributeExprLists(hashExprs);
+ }
+ TupleDescriptor outputTupleDesc = inputNode.getOutputTupleDesc();
+ updateTupleIds(outputTupleDesc);
+ }
+
+ public void updateTupleIds(TupleDescriptor outputTupleDesc) {
+ if (outputTupleDesc != null) {
+ clearTupleIds();
+ tupleIds.add(outputTupleDesc.getId());
+ } else {
+ clearTupleIds();
+ tupleIds.addAll(getChild(0).getOutputTupleIds());
+ }
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ // FE-planned LocalExchangeNode itself must stay non-serial. In the BE-planned path,
+ // the serial semantics belong to the upstream scan/exchange pipeline rather than the
+ // downstream LocalExchangeSource pipeline. Marking LocalExchangeNode as serial would
+ // incorrectly reduce the downstream pipeline's task count to 1.
+ msg.setIsSerialOperator(false);
+
+ msg.node_type = TPlanNodeType.LOCAL_EXCHANGE_NODE;
+ msg.local_exchange_node = new TLocalExchangeNode(exchangeType.toThrift());
+
+ if (exchangeType.isHashShuffle()) {
+ List<TExpr> thriftDistributeExprLists = new ArrayList<>();
+ for (Expr expr : distributeExprLists()) {
+ thriftDistributeExprLists.add(ExprToThriftVisitor.treeToThrift(expr));
+ }
+ msg.local_exchange_node.setDistributeExprLists(thriftDistributeExprLists);
+ }
+ }
+
+ private List<Expr> distributeExprLists() {
+ if (distributeExprLists == null) {
+ return Collections.emptyList();
+ }
+ return distributeExprLists;
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ return prefix + "type: " + exchangeType.name() + "\n";
+ }
+
+ public LocalExchangeType getExchangeType() {
+ return exchangeType;
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
+
+ /** LocalExchangeTypeRequire */
+ public interface LocalExchangeTypeRequire {
+ boolean satisfy(LocalExchangeType provide);
+
+ LocalExchangeType preferType();
+
+ default LocalExchangeTypeRequire autoRequireHash() {
+ return RequireHash.INSTANCE;
+ }
+
+ static NoRequire noRequire() {
+ return NoRequire.INSTANCE;
+ }
+
+ static RequireHash requireHash() {
+ return RequireHash.INSTANCE;
+ }
+
+ static RequireSpecific requirePassthrough() {
+ return requireSpecific(LocalExchangeType.PASSTHROUGH);
+ }
+
+ static RequireSpecific requirePassToOne() {
+ return requireSpecific(LocalExchangeType.PASS_TO_ONE);
+ }
+
+ static RequireSpecific requireBroadcast() {
+ return requireSpecific(LocalExchangeType.BROADCAST);
+ }
+
+ static RequireSpecific requireAdaptivePassthrough() {
+ return requireSpecific(LocalExchangeType.ADAPTIVE_PASSTHROUGH);
+ }
+
+ static RequireSpecific requireBucketHash() {
+ return requireSpecific(LocalExchangeType.BUCKET_HASH_SHUFFLE);
+ }
+
+ static RequireSpecific requireGlobalExecutionHash() {
+ return requireSpecific(LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE);
+ }
+
+ static RequireSpecific requireSpecific(LocalExchangeType require) {
+ return new RequireSpecific(require);
+ }
+
+ default LocalExchangeType noopTo(LocalExchangeType defaultType) {
+ LocalExchangeType preferType = preferType();
+ return (preferType == LocalExchangeType.NOOP) ? defaultType : preferType;
+ }
+ }
+
+ /** NoRequire */
+ public static class NoRequire implements LocalExchangeTypeRequire {
+ public static final NoRequire INSTANCE = new NoRequire();
+
+ @Override
+ public boolean satisfy(LocalExchangeType provide) {
+ return true;
+ }
+
+ @Override
+ public LocalExchangeType preferType() {
+ return LocalExchangeType.NOOP;
+ }
+ }
+
+ /** RequireHash */
+ public static class RequireHash implements LocalExchangeTypeRequire {
+ public static final RequireHash INSTANCE = new RequireHash();
+
+ @Override
+ public boolean satisfy(LocalExchangeType provide) {
+ switch (provide) {
+ case GLOBAL_EXECUTION_HASH_SHUFFLE:
+ case LOCAL_EXECUTION_HASH_SHUFFLE:
+ case BUCKET_HASH_SHUFFLE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @Override
+ public LocalExchangeType preferType() {
+ return LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE;
+ }
+
+ @Override
+ public LocalExchangeTypeRequire autoRequireHash() {
+ return this;
+ }
+ }
+
+ public static class RequireSpecific implements LocalExchangeTypeRequire {
+ LocalExchangeType requireType;
+
+ public RequireSpecific(LocalExchangeType requireType) {
+ this.requireType = requireType;
+ }
+
+ @Override
+ public boolean satisfy(LocalExchangeType provide) {
+ if (requireType == provide) {
+ return true;
+ }
+ // ADAPTIVE_PASSTHROUGH is a superset of PASSTHROUGH — both fan out data
+ // from fewer to more tasks. BE's need_to_local_exchange treats them as
+ // compatible, so ADAPTIVE_PASSTHROUGH satisfies a PASSTHROUGH requirement.
+ if (requireType == LocalExchangeType.PASSTHROUGH
+ && provide == LocalExchangeType.ADAPTIVE_PASSTHROUGH) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public LocalExchangeType preferType() {
+ return requireType;
+ }
+
+ @Override
+ public LocalExchangeTypeRequire autoRequireHash() {
+ if (requireType == LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE
+ || requireType == LocalExchangeType.BUCKET_HASH_SHUFFLE) {
+ return this;
+ }
+ return RequireHash.INSTANCE;
+ }
+ }
+
+ public enum LocalExchangeType {
+ NOOP,
+ GLOBAL_EXECUTION_HASH_SHUFFLE,
+ LOCAL_EXECUTION_HASH_SHUFFLE,
+ BUCKET_HASH_SHUFFLE,
+ PASSTHROUGH,
+ ADAPTIVE_PASSTHROUGH,
+ BROADCAST,
+ PASS_TO_ONE,
+ LOCAL_MERGE_SORT;
+
+ public boolean isHashShuffle() {
+ switch (this) {
+ case GLOBAL_EXECUTION_HASH_SHUFFLE:
+ case LOCAL_EXECUTION_HASH_SHUFFLE:
+ case BUCKET_HASH_SHUFFLE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ // Mirrors BE Pipeline::heavy_operations_on_the_sink():
+ // HASH_SHUFFLE, BUCKET_HASH_SHUFFLE, and ADAPTIVE_PASSTHROUGH perform
+ // heavy computation on the sink side. When the upstream pipeline has only
+ // 1 task (serial/pooling scan), a PASSTHROUGH fan-out must be inserted
+ // before these exchanges to avoid a single-task bottleneck.
+ public boolean isHeavyOperation() {
+ switch (this) {
+ case GLOBAL_EXECUTION_HASH_SHUFFLE:
+ case LOCAL_EXECUTION_HASH_SHUFFLE:
+ case BUCKET_HASH_SHUFFLE:
+ case ADAPTIVE_PASSTHROUGH:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public TLocalPartitionType toThrift() {
+ switch (this) {
+ case GLOBAL_EXECUTION_HASH_SHUFFLE:
+ return TLocalPartitionType.GLOBAL_EXECUTION_HASH_SHUFFLE;
+ case LOCAL_EXECUTION_HASH_SHUFFLE:
+ return TLocalPartitionType.LOCAL_EXECUTION_HASH_SHUFFLE;
+ case BUCKET_HASH_SHUFFLE:
+ return TLocalPartitionType.BUCKET_HASH_SHUFFLE;
+ case PASSTHROUGH:
+ return TLocalPartitionType.PASSTHROUGH;
+ case ADAPTIVE_PASSTHROUGH:
+ return TLocalPartitionType.ADAPTIVE_PASSTHROUGH;
+ case BROADCAST:
+ return TLocalPartitionType.BROADCAST;
+ case PASS_TO_ONE:
+ return TLocalPartitionType.PASS_TO_ONE;
+ case LOCAL_MERGE_SORT:
+ return TLocalPartitionType.LOCAL_MERGE_SORT;
+ default: {
+ throw new IllegalStateException("Unsupported LocalExchangeType: " + this);
+ }
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
index 3a21ae34868..40c75b90b13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
@@ -22,6 +22,10 @@ import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ColumnToThrift;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.resource.computegroup.ComputeGroup;
import org.apache.doris.system.Backend;
@@ -227,7 +231,17 @@ public class MaterializationNode extends PlanNode {
}
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return true;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ Pair<PlanNode, LocalExchangeType> enforceResult = enforceRequire(
+ translatorContext, children.get(0), 0, LocalExchangeTypeRequire.requirePassthrough());
+ children = new ArrayList<>();
+ children.add(enforceResult.first);
+ return Pair.of(this, LocalExchangeType.PASSTHROUGH);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
index daf65dc74d5..99c52013b5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
@@ -23,11 +23,17 @@ import org.apache.doris.analysis.JoinOperator;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TNestedLoopJoinNode;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
+import com.google.common.collect.Lists;
+
import java.util.List;
/**
@@ -164,8 +170,64 @@ public class NestedLoopJoinNode extends JoinNodeBase {
* Probe-side must have full data so join is a serial operator.
*/
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return joinOp == JoinOperator.RIGHT_OUTER_JOIN || joinOp == JoinOperator.RIGHT_ANTI_JOIN
|| joinOp == JoinOperator.RIGHT_SEMI_JOIN || joinOp == JoinOperator.FULL_OUTER_JOIN;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+
+ // Pooling mode: the fragment uses serial source (pooling scan or serial exchange).
+ // NLJ build side needs BROADCAST in pooling mode so all probe tasks see full build data.
+ boolean childUsePoolingScan = fragment.useSerialSource(translatorContext.getConnectContext());
+
+ LocalExchangeTypeRequire probeSideRequire;
+ LocalExchangeTypeRequire buildSideRequire;
+ LocalExchangeType outputType;
+ if (joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN) {
+ probeSideRequire = buildSideRequire = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ } else if (isSerialNode()) {
+ // RIGHT_OUTER/RIGHT_SEMI/RIGHT_ANTI/FULL_OUTER: probe side must be serial (1 task).
+ // Build side: noRequire() — inserting BROADCAST would inflate build pipeline's
+ // num_tasks while probe stays at 1, crashing in set_ready_to_read().
+ probeSideRequire = LocalExchangeTypeRequire.noRequire();
+ buildSideRequire = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ } else if (childUsePoolingScan) {
+ probeSideRequire = LocalExchangeTypeRequire.requireAdaptivePassthrough();
+ buildSideRequire = LocalExchangeTypeRequire.requireBroadcast();
+ outputType = LocalExchangeType.ADAPTIVE_PASSTHROUGH;
+ } else {
+ probeSideRequire = LocalExchangeTypeRequire.requireAdaptivePassthrough();
+ buildSideRequire = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.ADAPTIVE_PASSTHROUGH;
+ }
+
+ // Both sides use enforceRequire — it handles serial flag propagation, satisfy
+ // check (skip LE when child already outputs the required type, e.g., chained NLJs),
+ // serial ancestor skip, and serial child fallback (auto-upgrade noRequire to
+ // requirePassthrough when child is serial but this node is not).
+ PlanNode probeSide = enforceRequire(
+ translatorContext, children.get(0), 0, probeSideRequire).first;
+ PlanNode buildSide = enforceRequire(
+ translatorContext, children.get(1), 1, buildSideRequire).first;
+ this.children = Lists.newArrayList(probeSide, buildSide);
+ return Pair.of(this, outputType);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ // Build side (child 1) is a separate pipeline in BE. Normally,
+ // the serial-ancestor flag should be reset across pipeline boundaries.
+ // BUT when NLJ itself is serial (RIGHT_OUTER/ANTI/SEMI/FULL_OUTER),
+ // the probe pipeline has num_tasks=1. If we reset the flag, the
+ // build-side Exchange may insert PASSTHROUGH (restoring num_tasks to
+ // _num_instances), creating more build tasks than probe tasks. The
+ // extra build tasks have a NLJ shared state with empty source_deps,
+ // crashing in set_ready_to_read().
+ return childIndex == 1 && !isSerialNode();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 6ab25dcce02..6531b2634d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -59,6 +59,8 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.trees.plans.ScoreRangeInfo;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.planner.normalize.Normalizer;
import org.apache.doris.planner.normalize.PartitionRangePredicateNormalizer;
import org.apache.doris.qe.ConnectContext;
@@ -1060,6 +1062,9 @@ public class OlapScanNode extends ScanNode {
if (isPointQuery()) {
output.append(prefix).append("SHORT-CIRCUIT\n");
}
+ if (fragment.useSerialSource(ConnectContext.get())) {
+ output.append(prefix).append("POOLING-SCAN\n");
+ }
printNestedColumns(output, prefix, getTupleDesc());
@@ -1430,4 +1435,17 @@ public class OlapScanNode extends ScanNode {
}
return new OlapTableStreamUpdate(prev, next);
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent,
+ LocalExchangeTypeRequire parentRequire) {
+ boolean useSerialSource = fragment != null
+ && fragment.useSerialSource(translatorContext.getConnectContext());
+ if (useSerialSource) {
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
+ // Non-pooling OlapScan has bucket distribution — each instance scans specific buckets
+ return Pair.of(this, LocalExchangeType.BUCKET_HASH_SHUFFLE);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
index ed587df3816..a68d5567ebf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
@@ -22,8 +22,12 @@ import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.SortInfo;
import org.apache.doris.analysis.ToSqlParams;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.trees.plans.PartitionTopnPhase;
import org.apache.doris.nereids.trees.plans.WindowFuncType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPartTopNPhase;
import org.apache.doris.thrift.TPartitionSortNode;
@@ -169,4 +173,31 @@ public class PartitionSortNode extends PlanNode {
partitionSortNode.setPtopnPhase(pTopNPhase);
msg.partition_sort_node = partitionSortNode;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ LocalExchangeTypeRequire requireChild;
+ LocalExchangeType outputType;
+ if (phase == PartitionTopnPhase.TWO_PHASE_GLOBAL_PTOPN) {
+ // Use requireHash() so resolveExchangeType() can downgrade to LOCAL_EXECUTION_HASH_SHUFFLE,
+ // matching BE-native behavior where _use_serial_source=true causes LOCAL (not GLOBAL) hash.
+ // Output type is derived from the child's actual output (may be LOCAL or GLOBAL depending
+ // on whether a new exchange was inserted or the existing upstream exchange already satisfied).
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ outputType = null;
+ } else {
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ outputType = LocalExchangeType.PASSTHROUGH;
+ }
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, requireChild);
+ this.children = Lists.newArrayList(enforceResult.first);
+ return Pair.of(this, outputType != null ? outputType : enforceResult.second);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
index a46e2240f4c..98621ccc4f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
@@ -514,7 +514,7 @@ public class PlanFragment extends TreeNode<PlanFragment> {
// parallelism of non-serial operators.
// For bucket shuffle / colocate join fragment, always use serial source if the bucket scan nodes are
// serial.
- && (hasSerialScanNode() || (sink instanceof DataStreamSink && !planRoot.isSerialOperator()
+ && (hasSerialScanNode() || (sink instanceof DataStreamSink && !planRoot.isSerialNode()
&& planRoot.hasSerialChildren()));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 6e56919e4ff..3a2438a85dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -37,6 +37,9 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.planner.normalize.ExprNormalizeVisitor;
import org.apache.doris.planner.normalize.Normalizer;
import org.apache.doris.qe.ConnectContext;
@@ -142,7 +145,8 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
protected int nereidsId = -1;
- private List<List<Expr>> childrenDistributeExprLists = new ArrayList<>();
+ protected List<List<Expr>> childrenDistributeExprLists = new ArrayList<>();
+ protected List<Expr> distributeExprLists = new ArrayList<>();
protected PlanNode(PlanNodeId id, List<TupleId> tupleIds, String planNodeName) {
this.id = id;
@@ -467,7 +471,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
TPlanNode msg = new TPlanNode();
msg.node_id = id.asInt();
msg.setNereidsId(nereidsId);
- msg.setIsSerialOperator(isSerialOperator() && fragment.useSerialSource(ConnectContext.get()));
+ msg.setIsSerialOperator(isSerialOperatorOnBe(ConnectContext.get()));
msg.num_children = children.size();
msg.limit = limit;
for (TupleId tid : tupleIds) {
@@ -825,13 +829,20 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
}
// Operators need to be executed serially. (e.g. finalized agg without key)
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return false;
}
+ // Whether this node will have is_serial_operator=true on BE.
+ // Must match the condition in toThrift()/treeToThriftHelper().
+ // Subclasses (ExchangeNode) override to add hasSerialScanNode().
+ public boolean isSerialOperatorOnBe(ConnectContext context) {
+ return fragment != null && isSerialNode() && fragment.useSerialSource(context);
+ }
+
public boolean hasSerialChildren() {
if (children.isEmpty()) {
- return isSerialOperator();
+ return isSerialNode();
}
return children.stream().allMatch(PlanNode::hasSerialChildren);
}
@@ -933,4 +944,166 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
}
return StringUtils.join(mergeDisplayAccessPaths, ", ");
}
+
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ ArrayList<PlanNode> newChildren = Lists.newArrayList();
+ for (int i = 0; i < children.size(); i++) {
+ Pair<PlanNode, LocalExchangeType> childOutput
+ = enforceRequire(translatorContext, children.get(i), i, LocalExchangeTypeRequire.noRequire());
+ newChildren.add(childOutput.first);
+ }
+ this.children = newChildren;
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
+
+ /**
+ * Unified framework method: propagate serial flag → recurse child → satisfy check → Layer 1 skip → insert LE.
+ * Replaces the old enforceChild/enforceChildExchange/forceEnforceChildExchange trio.
+ *
+ * Layer 1 (shouldSkipLE): mirrors BE's need_to_local_exchange — skip when this node or
+ * an ancestor in the same pipeline is serial (operators[idx..end] has serial → skip).
+ * Layer 2 (require/output): each Node declares require and output in enforceAndDeriveLocalExchange.
+ */
+ protected Pair<PlanNode, LocalExchangeType> enforceRequire(
+ PlanTranslatorContext translatorContext, PlanNode child, int childIndex,
+ LocalExchangeTypeRequire require) {
+ // 1. Propagate serial-ancestor flag to child.
+ // For pipeline-splitting operators (shouldReset=true, e.g. non-streaming AGG):
+ // Drop inherited serial flag from parent (parent is in a different pipeline),
+ // but keep this node's own serial status (child is in the same pipeline as this
+ // node's sink, e.g. Exchange is in AGG_Sink pipeline).
+ // For non-splitting operators (shouldReset=false, e.g. streaming AGG):
+ // Inherit parent's serial flag + this node's own.
+ boolean inheritedSerial = shouldResetSerialFlagForChild(childIndex)
+ ? false : translatorContext.hasSerialAncestorInPipeline(this);
+ boolean childHasSerialAncestor = inheritedSerial || isSerialNode();
+ translatorContext.setHasSerialAncestorInPipeline(child, childHasSerialAncestor);
+
+ // 2. Recurse child (Layer 2: child declares its own require/output)
+ Pair<PlanNode, LocalExchangeType> childOutput =
+ child.enforceAndDeriveLocalExchange(translatorContext, this, require);
+
+ // 2.5. Serial child override: if child is serial on BE, force output to NOOP.
+ // This ensures the framework handles LE insertion uniformly — child nodes
+ // don't need to check serial status themselves. The serial child's actual
+ // distribution is irrelevant because it runs with 1 task; downstream needs
+ // LE to restore parallelism (step 3) or skip LE entirely (step 4b).
+ if (childOutput.first.isSerialOperatorOnBe(translatorContext.getConnectContext())) {
+ childOutput = Pair.of(childOutput.first, LocalExchangeType.NOOP);
+ }
+
+ // 3. Framework-level serial child check (mirrors BE base class required_data_distribution):
+ // If child will be serial on BE but this node is not serial, the pipeline has a
+ // 1-task serial child feeding an N-task non-serial parent. Without LE, pipeline
+ // splits (AGG/JOIN) create paired pipelines with mismatched num_tasks → crash.
+ // Upgrade noRequire to requirePassthrough so LE is inserted to restore parallelism.
+ if (require instanceof LocalExchangeNode.NoRequire
+ && childOutput.first.isSerialOperatorOnBe(translatorContext.getConnectContext())
+ && !isSerialOperatorOnBe(translatorContext.getConnectContext())) {
+ require = LocalExchangeTypeRequire.requirePassthrough();
+ }
+
+ // 4. Satisfy check: child output meets requirement → done
+ if (require.satisfy(childOutput.second)) {
+ return childOutput;
+ }
+
+ // 4. Layer 1: skip LE when serial operator or ancestor in same pipeline
+ // Equivalent to BE's need_to_local_exchange: any_of(operators[idx..end], is_serial) → skip
+ if (translatorContext.hasSerialAncestorInPipeline(this) || isSerialNode()) {
+ return childOutput;
+ }
+
+ // 5. Resolve exchange type and create LE node
+ LocalExchangeType preferType = AddLocalExchange.resolveExchangeType(
+ require, translatorContext, this, childOutput.first);
+ List<Expr> distributeExprs = getChildDistributeExprList(childIndex);
+ PlanNode leNode = createLocalExchange(translatorContext, childOutput.first, preferType, distributeExprs);
+ return Pair.of(leNode, preferType);
+ }
+
+ /**
+ * Create a LocalExchangeNode wrapping child with the given exchange type.
+ * No child-type skip — matches BE's _add_local_exchange which inserts LE for any child
+ * type without checking instanceof.
+ *
+ * Handles heavy-ops bottleneck avoidance (mirrors BE pipeline_fragment_context.cpp):
+ * when upstream has 1 task (serial source) and exchange is heavy (hash/bucket/adaptive),
+ * insert a PASSTHROUGH fan-out first to avoid single-task bottleneck on the heavy
+ * exchange sink. Only applies to local-shuffle (pooling scan) fragments.
+ */
+ protected PlanNode createLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode child, LocalExchangeType exchangeType, List<Expr> distributeExprs) {
+ if (fragment != null && fragment.useSerialSource(translatorContext.getConnectContext())
+ && exchangeType.isHeavyOperation() && child.isSerialNode()) {
+ PlanNode ptNode = new LocalExchangeNode(translatorContext.nextPlanNodeId(),
+ child, LocalExchangeType.PASSTHROUGH, null);
+ return new LocalExchangeNode(translatorContext.nextPlanNodeId(), ptNode,
+ exchangeType, distributeExprs);
+ }
+ return new LocalExchangeNode(translatorContext.nextPlanNodeId(), child,
+ exchangeType, distributeExprs);
+ }
+
+ /**
+ * Whether the child at {@code childIndex} starts a new pipeline context, causing
+ * its serial-ancestor flag to be reset to {@code false} rather than inherited from this node.
+ * Override to return {@code true} for pipeline-splitting nodes (LocalExchangeNode) and nodes
+ * whose children run in an independent pipeline segment (SortNode above analytic, etc.).
+ */
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return false;
+ }
+
+ protected List<Expr> getChildDistributeExprList(int childIndex) {
+ if ((childrenDistributeExprLists == null || childrenDistributeExprLists.size() <= childIndex)) {
+ return null;
+ } else {
+ return childrenDistributeExprLists.get(childIndex);
+ }
+ }
+
+ /**
+ * Returns the operator's own semantically-defined partition expressions
+ * (e.g. GROUP BY exprs for aggregation, PARTITION BY exprs for analytic).
+ * Corresponds to BE's fallback path: tnode.agg_node.grouping_exprs /
+ * tnode.analytic_node.partition_exprs when _followed_by_shuffled_operator=false.
+ * Override in subclasses that have intrinsic partition keys.
+ */
+ protected List<Expr> getSemanticPartitionExprs() {
+ return null;
+ }
+
+ /**
+ * Returns true if there are effective (non-empty) partition expressions,
+ * mirroring BE's _partition_exprs logic:
+ * _followed_by_shuffled_operator=true → distribute_expr_lists[0] (child distribute key)
+ * _followed_by_shuffled_operator=false → semantic partition exprs (grouping / partition by)
+ * parentRequire.preferType().isHashShuffle() corresponds to _followed_by_shuffled_operator=true.
+ */
+ protected boolean hasPartitionExprs(LocalExchangeTypeRequire parentRequire) {
+ if (parentRequire.preferType().isHashShuffle()) {
+ List<Expr> childExprs = getChildDistributeExprList(0);
+ return childExprs != null && !childExprs.isEmpty();
+ }
+ List<Expr> semanticExprs = getSemanticPartitionExprs();
+ return semanticExprs != null && !semanticExprs.isEmpty();
+ }
+
+ public List<List<Expr>> getChildrenDistributeExprLists() {
+ return childrenDistributeExprLists;
+ }
+
+ public List<Expr> getDistributeExprLists() {
+ return distributeExprLists;
+ }
+
+ public void setDistributeExprLists(List<Expr> distributeExprLists) {
+ if (distributeExprLists == null) {
+ this.distributeExprLists = Collections.emptyList();
+ } else {
+ this.distributeExprLists = distributeExprLists;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
index 838facd27b3..3a7bdd63745 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
@@ -21,6 +21,10 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -29,6 +33,7 @@ import org.apache.doris.thrift.TRecCTENode;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Lists;
+import java.util.ArrayList;
import java.util.List;
public class RecursiveCteNode extends PlanNode {
@@ -86,4 +91,23 @@ public class RecursiveCteNode extends PlanNode {
.add("tid", tupleIds.get(0).asInt())
.add("isUnionAll", isUnionAll).toString();
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ ArrayList<PlanNode> newChildren = Lists.newArrayList();
+ for (int i = 0; i < children.size(); i++) {
+ PlanNode child = children.get(i);
+ Pair<PlanNode, LocalExchangeType> childOutput = enforceRequire(
+ translatorContext, child, i, LocalExchangeTypeRequire.noRequire());
+ newChildren.add(childOutput.first);
+ }
+ this.children = newChildren;
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
index c0ed988010c..9f98f7b2c27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -21,6 +21,10 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -67,7 +71,13 @@ public class RecursiveCteScanNode extends PlanNode {
}
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return true;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java
index e28094cfe89..bf1be174f97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java
@@ -19,6 +19,10 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.GroupingInfo;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -86,11 +90,21 @@ public class RepeatNode extends PlanNode {
// Determined by its child.
@Override
- public boolean isSerialOperator() {
- return children.get(0).isSerialOperator();
+ public boolean isSerialNode() {
+ return children.get(0).isSerialNode();
}
public GroupingInfo getGroupingInfo() {
return groupingInfo;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, parentRequire);
+ children = new java.util.ArrayList<>();
+ children.add(enforceResult.first);
+ return Pair.of(this, enforceResult.second);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
index f4ea16c9360..e1860e5bcd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
@@ -263,7 +263,7 @@ public final class RuntimeFilter {
for (RuntimeFilterTarget target : targets) {
tFilter.putToPlanIdToTargetExpr(target.node.getId().asInt(), ExprToThriftVisitor.treeToThrift(target.expr));
hasSerialTargets = hasSerialTargets
- || (target.node.isSerialOperator() && target.node.fragment.useSerialSource(ConnectContext.get()));
+ || target.node.isSerialOperatorOnBe(ConnectContext.get());
}
boolean enableSyncFilterSize = ConnectContext.get() != null
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 02a40761db9..e69d3101548 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -42,11 +42,15 @@ import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.cloud.catalog.CloudPartition;
import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.system.Backend;
@@ -733,7 +737,7 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
}
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
ConnectContext context = ConnectContext.get();
if (context == null) {
return numScanBackends() <= 0;
@@ -747,7 +751,15 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
@Override
public boolean hasSerialScanChildren() {
- return isSerialOperator();
+ return isSerialNode();
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ // Base ScanNode returns NOOP — only OlapScanNode overrides with BUCKET_HASH_SHUFFLE
+ // for non-pooling scans that have bucket distribution.
+ return Pair.of(this, LocalExchangeType.NOOP);
}
public void setDesc(TupleDescriptor desc) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
index 2ef0b364be1..92b6975afcf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
@@ -20,6 +20,10 @@
package org.apache.doris.planner;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -57,7 +61,17 @@ public class SelectNode extends PlanNode {
// Determined by its child.
@Override
- public boolean isSerialOperator() {
- return children.get(0).isSerialOperator();
+ public boolean isSerialNode() {
+ return children.get(0).isSerialNode();
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, parentRequire);
+ this.children = new ArrayList<>();
+ this.children.add(enforceResult.first);
+ return Pair.of(this, enforceResult.second);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
index aecb6e43c88..a7d89278f0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
@@ -22,6 +22,10 @@ import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExceptNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
@@ -37,6 +41,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -193,4 +198,48 @@ public abstract class SetOperationNode extends PlanNode {
public boolean isBucketShuffle() {
return distributionMode.equals(DistributionMode.BUCKET_SHUFFLE);
}
+
+ public boolean isColocate() {
+ return isColocate;
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ LocalExchangeTypeRequire requireChild;
+ LocalExchangeType outputType;
+ PlanNode firstChild = children.isEmpty() ? null : children.get(0);
+ if (this instanceof UnionNode) {
+ // Propagate parent's hash requirement to children when parent requires hash distribution.
+ // Matches BE's UnionSinkOperatorX which returns GLOBAL_HASH(_distribute_exprs) whenever
+ // _followed_by_shuffled_operator=true, regardless of whether _distribute_exprs is empty.
+ boolean canPropagateHash = parentRequire.preferType().isHashShuffle();
+ requireChild = canPropagateHash ? parentRequire : LocalExchangeTypeRequire.noRequire();
+ outputType = canPropagateHash
+ ? AddLocalExchange.resolveExchangeType(requireChild, translatorContext, this, firstChild)
+ : LocalExchangeType.NOOP;
+ } else {
+ // Intersect / Except
+ if (AddLocalExchange.isColocated(this)) {
+ requireChild = LocalExchangeTypeRequire.requireBucketHash();
+ outputType = LocalExchangeType.BUCKET_HASH_SHUFFLE;
+ } else {
+ requireChild = parentRequire.autoRequireHash();
+ outputType = AddLocalExchange.resolveExchangeType(
+ requireChild, translatorContext, this, firstChild);
+ }
+ }
+
+ ArrayList<PlanNode> newChildren = Lists.newArrayList();
+ for (int i = 0; i < children.size(); i++) {
+ newChildren.add(enforceRequire(translatorContext, children.get(i), i, requireChild).first);
+ }
+ this.children = newChildren;
+ return Pair.of(this, outputType);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
index c94c1764d61..6c984dbed4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
@@ -25,6 +25,9 @@ import org.apache.doris.analysis.ExprToSqlVisitor;
import org.apache.doris.analysis.SortInfo;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@@ -217,7 +220,7 @@ public class SortNode extends PlanNode {
// If it's analytic sort or not merged by a followed exchange node, it must output the global ordered data.
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return !isAnalyticSort && !mergeByexchange;
}
@@ -245,4 +248,46 @@ public class SortNode extends PlanNode {
List<Pair<Integer, Integer>> topnFilterTargets) {
this.topnFilterTargets = topnFilterTargets;
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(PlanTranslatorContext translatorContext,
+ PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+
+ LocalExchangeTypeRequire requireChild;
+ LocalExchangeType outputType = null;
+ if (isAnalyticSort) {
+ // BE: SortSink._is_analytic_sort=true → required_data_distribution() = HASH.
+ // This sort serves a parent AnalyticEvalNode (window function) and requires
+ // data partitioned by the analytic's partition exprs.
+ if (AddLocalExchange.isColocated(this)) {
+ requireChild = LocalExchangeTypeRequire.requireHash();
+ outputType = AddLocalExchange.resolveExchangeType(
+ LocalExchangeTypeRequire.requireHash(), translatorContext, this,
+ children.get(0));
+ } else {
+ requireChild = parentRequire.autoRequireHash();
+ }
+ } else if (mergeByexchange) {
+ // BE: SortSink._merge_by_exchange=true → required_data_distribution() = PASSTHROUGH.
+ requireChild = LocalExchangeTypeRequire.requirePassthrough();
+ outputType = LocalExchangeType.PASSTHROUGH;
+ } else {
+ // BE: else → NOOP
+ requireChild = LocalExchangeTypeRequire.noRequire();
+ outputType = LocalExchangeType.NOOP;
+ }
+
+ Pair<PlanNode, LocalExchangeType> enforceResult
+ = enforceRequire(translatorContext, children.get(0), 0, requireChild);
+ this.children = Lists.newArrayList(enforceResult.first);
+ if (outputType == null) {
+ outputType = enforceResult.second;
+ }
+ return Pair.of(this, outputType);
+ }
+
+ @Override
+ protected boolean shouldResetSerialFlagForChild(int childIndex) {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
index 3630bfd72a9..c363b30c318 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
@@ -23,6 +23,10 @@ import org.apache.doris.analysis.ExprToThriftVisitor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@@ -120,4 +124,24 @@ public class TableFunctionNode extends PlanNode {
msg.table_function_node.addToOutputSlotIds(slotId.asInt());
}
}
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ // Mirrors BE TableFunctionOperatorX::required_data_distribution() which always
+ // returns PASSTHROUGH, regardless of child's serial status.
+ //
+ // Conceptual model: TableFunction requires PASSTHROUGH input but outputs
+ // "unknown distribution" (NOOP). This means downstream operators (e.g. Sort)
+ // must independently evaluate their own requirements against NOOP, naturally
+ // triggering exchange insertion when they require PASSTHROUGH.
+ //
+ // In BE, need_to_local_exchange() Step 4 treats non-hash exchanges (PASSTHROUGH)
+ // as always needing insertion, so "PASSTHROUGH doesn't satisfy PASSTHROUGH" —
+ // which is equivalent to our FE model of require=PASSTHROUGH, output=NOOP.
+ Pair<PlanNode, LocalExchangeType> enforceResult = enforceRequire(
+ translatorContext, children.get(0), 0, LocalExchangeTypeRequire.requirePassthrough());
+ children = Lists.newArrayList(enforceResult.first);
+ return Pair.of(this, LocalExchangeType.NOOP);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java
index 36e50b8f501..0b05cafd9cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java
@@ -37,7 +37,7 @@ public class UnionNode extends SetOperationNode {
// If it is a union without children which means it will output some constant values, we should use a serial union
// to output non-duplicated data.
@Override
- public boolean isSerialOperator() {
+ public boolean isSerialNode() {
return children.isEmpty();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 5181a77b042..49ca93b7d7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -352,7 +352,6 @@ public class Coordinator implements CoordInterface {
} else {
distributedPlans = ((NereidsPlanner) planner).getDistributedPlans();
}
-
setFromUserProperty(context);
this.queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()));
@@ -428,6 +427,8 @@ public class Coordinator implements CoordInterface {
this.queryOptions.setFeProcessUuid(ExecuteEnv.getInstance().getProcessUUID());
this.queryOptions.setMysqlRowBinaryFormat(
context.getCommand() == MysqlCommand.COM_STMT_EXECUTE);
+ // Old Coordinator never plans local exchange in FE. Force BE to plan its own.
+ this.queryOptions.setEnableLocalShufflePlanner(false);
}
public ConnectContext getConnectContext() {
@@ -2884,7 +2885,7 @@ public class Coordinator implements CoordInterface {
Optional<ScanNode> node = scanNodes.stream().filter(
scanNode -> scanNode.getId().asInt() == scanId).findFirst();
Preconditions.checkArgument(node.isPresent());
- FInstanceExecParam instanceParamToScan = node.get().isSerialOperator()
+ FInstanceExecParam instanceParamToScan = node.get().isSerialNode()
? firstInstanceParam : instanceParam;
if (!instanceParamToScan.perNodeScanRanges.containsKey(nodeScanRange.getKey())) {
range.put(nodeScanRange.getKey(), Lists.newArrayList());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index f8c7509f102..19c2eaec3ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -101,6 +101,7 @@ public class NereidsCoordinator extends Coordinator {
setForInsert(-1L);
}
+ syncLocalShufflePlannerOption();
Preconditions.checkState(!planner.getFragments().isEmpty()
&& coordinatorContext.instanceNum.get() > 0, "Fragment and Instance can not be empty˚");
}
@@ -115,6 +116,7 @@ public class NereidsCoordinator extends Coordinator {
// we don't need to check the dataSink, Because setting jobId means this must be a load operation
setForInsert(jobId);
+ syncLocalShufflePlannerOption();
Preconditions.checkState(!planner.getFragments().isEmpty()
&& coordinatorContext.instanceNum.get() > 0, "Fragment and Instance can not be empty˚");
}
@@ -132,11 +134,20 @@ public class NereidsCoordinator extends Coordinator {
// same reason in `setForInsert`
this.coordinatorContext.queryOptions.setDisableFileCache(true);
this.needEnqueue = false;
+ syncLocalShufflePlannerOption();
Preconditions.checkState(!fragments.isEmpty()
&& coordinatorContext.instanceNum.get() > 0, "Fragment and Instance can not be empty˚");
}
+ private void syncLocalShufflePlannerOption() {
+ coordinatorContext.queryOptions.setEnableLocalShufflePlanner(
+ coordinatorContext.distributedPlans != null
+ && !coordinatorContext.distributedPlans.isEmpty()
+ && coordinatorContext.connectContext != null
+ && coordinatorContext.connectContext.getSessionVariable().isEnableLocalShufflePlanner());
+ }
+
@Override
public void exec() throws Exception {
enqueue(coordinatorContext.connectContext);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 72670ec6f02..947109456da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -337,6 +337,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_LOCAL_SHUFFLE = "enable_local_shuffle";
+ public static final String ENABLE_LOCAL_SHUFFLE_PLANNER = "enable_local_shuffle_planner";
+
public static final String FORCE_TO_LOCAL_SHUFFLE = "force_to_local_shuffle";
public static final String ENABLE_LOCAL_MERGE_SORT = "enable_local_merge_sort";
@@ -1595,6 +1597,12 @@ public class SessionVariable implements Serializable, Writable {
"Whether to enable local shuffle on pipelineX engine."}, needForward = true)
private boolean enableLocalShuffle = true;
+ @VarAttrDef.VarAttr(
+ name = ENABLE_LOCAL_SHUFFLE_PLANNER, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
+ description = {"是否在FE规划Local Shuffle",
+ "Whether to plan local shuffle in frontend"}, needForward = true)
+ private boolean enableLocalShufflePlanner = true;
+
@VarAttrDef.VarAttr(
name = FORCE_TO_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
description = {"是否在 pipelineX 引擎上强制开启 local shuffle 优化",
@@ -4730,6 +4738,18 @@ public class SessionVariable implements Serializable, Writable {
this.enableLocalShuffle = enableLocalShuffle;
}
+ public boolean isEnableLocalShuffle() {
+ return enableLocalShuffle;
+ }
+
+ public boolean isEnableLocalShufflePlanner() {
+ return enableLocalShufflePlanner;
+ }
+
+ public void setEnableLocalShufflePlanner(boolean enableLocalShufflePlanner) {
+ this.enableLocalShufflePlanner = enableLocalShufflePlanner;
+ }
+
public boolean enablePushDownNoGroupAgg() {
return enablePushDownNoGroupAgg;
}
@@ -5633,6 +5653,8 @@ public class SessionVariable implements Serializable, Writable {
// Set Iceberg write target file size
tResult.setIcebergWriteTargetFileSizeBytes(icebergWriteTargetFileSizeBytes);
+ tResult.setEnableLocalShufflePlanner(enableLocalShufflePlanner);
+
return tResult;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 11d633a5644..92163eacc27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1110,8 +1110,10 @@ public class StmtExecutor {
// 1. CreateTableCommand(mainly for create as select).
// 2. LoadCommand.
// 3. InsertOverwriteTableCommand.
+ // 4. MergeIntoCommand (merge into ... using ...).
if ((plan instanceof Command) && !(plan instanceof LoadCommand)
- && !(plan instanceof CreateTableCommand) && !(plan instanceof InsertOverwriteTableCommand)) {
+ && !(plan instanceof CreateTableCommand) && !(plan instanceof InsertOverwriteTableCommand)
+ && !(plan instanceof org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand)) {
// Commands like SHOW QUERY PROFILE will not have profile.
return false;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index 4f8499970c9..7686dc98c24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -95,7 +95,6 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.function.BiConsumer;
import java.util.function.Supplier;
public class ThriftPlansBuilder {
@@ -280,7 +279,8 @@ public class ThriftPlansBuilder {
return workerCounter;
}
- private static Map<Integer, Integer> computeExchangeSenderNum(PipelineDistributedPlan distributedPlan) {
+ private static Map<Integer, Integer> computeExchangeSenderNum(
+ PipelineDistributedPlan distributedPlan) {
Map<Integer, Integer> senderNum = Maps.newLinkedHashMap();
for (Entry<ExchangeNode, DistributedPlan> kv : distributedPlan.getInputs().entries()) {
ExchangeNode exchangeNode = kv.getKey();
@@ -427,6 +427,25 @@ public class ThriftPlansBuilder {
TPlanFragment planThrift = fragment.toThrift();
planThrift.query_cache_param = fragment.queryCacheParam;
params.setFragment(planThrift);
+ if (planThrift.isSetPlan()) {
+ long leCount = planThrift.getPlan().getNodes().stream()
+ .filter(n -> n.getNodeType()
+ == org.apache.doris.thrift.TPlanNodeType.LOCAL_EXCHANGE_NODE)
+ .count();
+ boolean hasPooling = fragmentPlan.getInstanceJobs().stream().anyMatch(
+ org.apache.doris.nereids.trees.plans.distribute.worker.job
+ .LocalShuffleAssignedJob.class::isInstance);
+ if (hasPooling) {
+ org.apache.logging.log4j.LogManager.getLogger(ThriftPlansBuilder.class).info(
+ "ThriftPlan qid={} fragment={} leNodes={} pooling=true"
+ + " parallelInstances={} instances={}",
+ org.apache.doris.common.util.DebugUtil.printId(
+ coordinatorContext.queryId),
+ fragment.getFragmentId(), leCount,
+ params.isSetParallelInstances() ? params.getParallelInstances() : -1,
+ fragmentPlan.getInstanceJobs().size());
+ }
+ }
params.setLocalParams(Lists.newArrayList());
params.setWorkloadGroups(coordinatorContext.getWorkloadGroups());
@@ -583,39 +602,55 @@ public class ThriftPlansBuilder {
}
Map<Integer, Integer> destIdToInstanceId = Maps.newLinkedHashMap();
- filterInstancesWhichReceiveDataFromRemote(
- receivePlan, worker,
- (instanceJob, destId) -> destIdToInstanceId.put(destId, instanceToIndex.get(instanceJob))
- );
- return destIdToInstanceId;
- }
-
- private static void filterInstancesWhichReceiveDataFromRemote(
- PipelineDistributedPlan receivePlan, DistributedPlanWorker filterWorker,
- BiConsumer<AssignedJob, Integer> computeFn) {
-
- // current only support all input plans have same destination with same order,
- // so we can get first input plan to compute shuffle index to instance id
- Set<Entry<ExchangeNode, DistributedPlan>> exchangeToChildPlanSet = receivePlan.getInputs().entries();
- if (exchangeToChildPlanSet.isEmpty()) {
- return;
+ // dest id is defined by sender-side destination order rather than receiver fragment instance order.
+ // When FE inserts local shuffle or serial source, one worker may have multiple local receiver instances,
+ // but only the first one actually receives remote data. In that case receiverPlan.getInstanceJobs()
+ // no longer matches sender destinations, and using receiver instance order will produce an invalid
+ // shuffle_idx_to_instance_idx mapping for BE.
+ //
+ // When a fragment has multiple ExchangeNode inputs (e.g., NLJ with probe + BROADCAST build sides),
+ // always pick the one with the most destinations on this worker. A BROADCAST input has 1 dest per BE
+ // while the main data-carrying input (HASH-partitioned probe) has N dests per BE; using the BROADCAST
+ // one would produce a 1-entry map for GLOBAL_HASH LOCAL_EXCHANGE, causing rows to be lost.
+ Entry<ExchangeNode, DistributedPlan> exchangeToChildPlan = null;
+ int maxDestsOnWorker = -1;
+ for (Entry<ExchangeNode, DistributedPlan> entry : receivePlan.getInputs().entries()) {
+ ExchangeNode exchNode = entry.getKey();
+ PipelineDistributedPlan childPlan = (PipelineDistributedPlan) entry.getValue();
+ for (Entry<DataSink, List<AssignedJob>> kv : childPlan.getDestinations().entrySet()) {
+ if (kv.getKey().getExchNodeId().asInt() != exchNode.getId().asInt()) {
+ continue;
+ }
+ int destsOnWorker = (int) kv.getValue().stream()
+ .filter(j -> j.getAssignedWorker().id() == worker.id())
+ .count();
+ if (destsOnWorker > maxDestsOnWorker) {
+ maxDestsOnWorker = destsOnWorker;
+ exchangeToChildPlan = entry;
+ }
+ break;
+ }
+ }
+ if (exchangeToChildPlan == null) {
+ return destIdToInstanceId;
}
- Entry<ExchangeNode, DistributedPlan> exchangeToChildPlan = exchangeToChildPlanSet.iterator().next();
ExchangeNode linkNode = exchangeToChildPlan.getKey();
PipelineDistributedPlan firstInputPlan = (PipelineDistributedPlan) exchangeToChildPlan.getValue();
- Map<DataSink, List<AssignedJob>> sinkToDestInstances = firstInputPlan.getDestinations();
- for (Entry<DataSink, List<AssignedJob>> kv : sinkToDestInstances.entrySet()) {
+ for (Entry<DataSink, List<AssignedJob>> kv : firstInputPlan.getDestinations().entrySet()) {
DataSink senderSink = kv.getKey();
- if (senderSink.getExchNodeId().asInt() == linkNode.getId().asInt()) {
- for (int destId = 0; destId < kv.getValue().size(); destId++) {
- AssignedJob assignedJob = kv.getValue().get(destId);
- if (assignedJob.getAssignedWorker().id() == filterWorker.id()) {
- computeFn.accept(assignedJob, destId);
- }
+ if (senderSink.getExchNodeId().asInt() != linkNode.getId().asInt()) {
+ continue;
+ }
+ List<AssignedJob> destinationJobs = kv.getValue();
+ for (int destId = 0; destId < destinationJobs.size(); destId++) {
+ AssignedJob destinationJob = destinationJobs.get(destId);
+ if (destinationJob.getAssignedWorker().id() == worker.id()) {
+ destIdToInstanceId.put(destId, instanceToIndex.get(destinationJob));
}
- break;
}
+ break;
}
+ return destIdToInstanceId;
}
private static Set<Integer> setParamsForRecursiveCteNode(List<PipelineDistributedPlan> distributedPlans,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
index 0695b05ef70..f813988dd91 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java
@@ -51,6 +51,10 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
@@ -136,6 +140,12 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements MemoP
}
}
+ @Test
+ public void testSimpleCaseWithExistsSubqueryCanAnalyze() {
+ String sql = "select case exists(select k1 from t7) when true then 2 when false then 4 else 5 end a3 from t6";
+ Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).analyze(sql));
+ }
+
@Test
public void testWhereSql2AfterAnalyzed() {
// after analyze
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
new file mode 100644
index 00000000000..3bfaadee8be
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
@@ -0,0 +1,662 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.AssertNumRowsElement;
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.GroupingInfo;
+import org.apache.doris.analysis.JoinOperator;
+import org.apache.doris.analysis.OrderByElement;
+import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.SortInfo;
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.trees.plans.PartitionTopnPhase;
+import org.apache.doris.nereids.trees.plans.WindowFuncType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TPartitionType;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TScanRangeLocations;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class LocalShuffleNodeCoverageTest {
+ private static final AtomicInteger NEXT_ID = new AtomicInteger(1);
+
+ @Test
+ public void testSelectNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+
+ TrackingPlanNode childNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ SelectNode selectWithNoopChild = new SelectNode(nextPlanNodeId(), childNoop);
+ Pair<PlanNode, LocalExchangeType> output = selectWithNoopChild.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+
+ // resolveExchangeType with RequireHash always returns LOCAL_EXECUTION_HASH_SHUFFLE
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, output.second);
+ Assertions.assertEquals(LocalExchangeNode.RequireHash.class, childNoop.lastRequire.getClass());
+ assertChildLocalExchangeType(selectWithNoopChild, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+
+ TrackingPlanNode childBucket = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.BUCKET_HASH_SHUFFLE);
+ SelectNode selectWithBucketChild = new SelectNode(nextPlanNodeId(), childBucket);
+ Pair<PlanNode, LocalExchangeType> bucketOutput = selectWithBucketChild.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, bucketOutput.second);
+ Assertions.assertSame(childBucket, selectWithBucketChild.getChild(0));
+ }
+
+ @Test
+ public void testRepeatNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ GroupingInfo groupingInfo = Mockito.mock(GroupingInfo.class);
+ TupleDescriptor outputTuple = new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement()));
+ Mockito.when(groupingInfo.getOutputTupleDesc()).thenReturn(outputTuple);
+ Mockito.when(groupingInfo.getPreRepeatExprs()).thenReturn(Collections.emptyList());
+
+ TrackingPlanNode childNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ RepeatNode repeatNode = new RepeatNode(nextPlanNodeId(), childNoop, groupingInfo,
+ Collections.singletonList(Collections.emptySet()), Collections.emptySet(),
+ Collections.singletonList(Collections.emptyList()));
+ Pair<PlanNode, LocalExchangeType> output = repeatNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ // resolveExchangeType with RequireHash always returns LOCAL_EXECUTION_HASH_SHUFFLE
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, output.second);
+ assertChildLocalExchangeType(repeatNode, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+ }
+
+ @Test
+ public void testTableFunctionNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ TrackingPlanNode childNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TableFunctionNode tableFunctionNode = new TableFunctionNode(nextPlanNodeId(), childNoop,
+ new TupleId(NEXT_ID.getAndIncrement()), new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
+
+ // TableFunctionNode always requires PASSTHROUGH from child and outputs NOOP.
+ // This mirrors BE's TableFunctionOperatorX::required_data_distribution() override.
+ // Parent's requireHash is ignored — TableFunction's own PASSTHROUGH requirement takes precedence.
+ Pair<PlanNode, LocalExchangeType> output = tableFunctionNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, output.second);
+ assertChildLocalExchangeType(tableFunctionNode, 0, LocalExchangeType.PASSTHROUGH);
+ }
+
+ @Test
+ public void testPartitionSortNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ SortInfo sortInfo = Mockito.mock(SortInfo.class);
+ TupleDescriptor sortTuple = new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement()));
+ Mockito.when(sortInfo.getOrderingExprs()).thenReturn(Collections.emptyList());
+ Mockito.when(sortInfo.getIsAscOrder()).thenReturn(Collections.emptyList());
+ Mockito.when(sortInfo.getSortTupleDescriptor()).thenReturn(sortTuple);
+
+ TrackingPlanNode childNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ PartitionSortNode globalTopnNode = new PartitionSortNode(nextPlanNodeId(), childNoop,
+ WindowFuncType.ROW_NUMBER, Collections.emptyList(), sortInfo, false, 1,
+ PartitionTopnPhase.TWO_PHASE_GLOBAL_PTOPN);
+ Pair<PlanNode, LocalExchangeType> globalOutput = globalTopnNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ // enforceRequire resolves RequireHash to LOCAL_EXECUTION_HASH_SHUFFLE (FE-planned always uses LOCAL)
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, globalOutput.second);
+ assertChildLocalExchangeType(globalTopnNode, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+
+ TrackingPlanNode childNoop2 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ PartitionSortNode passthroughNode = new PartitionSortNode(nextPlanNodeId(), childNoop2,
+ WindowFuncType.ROW_NUMBER, Collections.emptyList(), sortInfo, false, 1,
+ PartitionTopnPhase.TWO_PHASE_LOCAL_PTOPN);
+ Pair<PlanNode, LocalExchangeType> passthroughOutput = passthroughNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, passthroughOutput.second);
+ assertChildLocalExchangeType(passthroughNode, 0, LocalExchangeType.PASSTHROUGH);
+ }
+
+ @Test
+ public void testMaterializationNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ TrackingPlanNode childNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TupleDescriptor tupleDescriptor = new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement()));
+ TestMaterializationNode node = new TestMaterializationNode(nextPlanNodeId(), tupleDescriptor, childNoop);
+
+ // MaterializationNode.isSerialNode() returns true → enforceChild skips exchange.
+ // Output is still PASSTHROUGH (hardcoded in MaterializationNode).
+ Pair<PlanNode, LocalExchangeType> output = node.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, output.second);
+ // Child is NOT wrapped in LocalExchangeNode because serial operator skips exchange.
+ Assertions.assertSame(childNoop, node.getChild(0));
+ }
+
+ @Test
+ public void testCteAndRecursiveNodesAndEmptySet() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+
+ CTEScanNode cteScanNode = new CTEScanNode(new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())),
+ ScanContext.EMPTY);
+ Pair<PlanNode, LocalExchangeType> cteOutput = cteScanNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, cteOutput.second);
+
+ RecursiveCteScanNode recursiveScanNode = new RecursiveCteScanNode("r", nextPlanNodeId(),
+ new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ Pair<PlanNode, LocalExchangeType> recursiveScanOutput = recursiveScanNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, recursiveScanOutput.second);
+
+ EmptySetNode emptySetNode = new EmptySetNode(nextPlanNodeId(),
+ new ArrayList<>(Collections.singletonList(new TupleId(NEXT_ID.getAndIncrement()))));
+ Pair<PlanNode, LocalExchangeType> emptyOutput = emptySetNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, emptyOutput.second);
+
+ TrackingPlanNode recursiveChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ RecursiveCteNode recursiveNode = new RecursiveCteNode(nextPlanNodeId(), new TupleId(NEXT_ID.getAndIncrement()),
+ "r", true);
+ recursiveNode.addChild(recursiveChild);
+ Pair<PlanNode, LocalExchangeType> recursiveOutput = recursiveNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, recursiveOutput.second);
+ Assertions.assertEquals(LocalExchangeNode.NoRequire.class, recursiveChild.lastRequire.getClass());
+ }
+
+ @Test
+ public void testHashJoinNodeBranches() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ List<Expr> eqConjuncts = Collections.singletonList(Mockito.mock(BinaryPredicate.class));
+
+ TrackingPlanNode probe = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode build = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ HashJoinNode broadcastJoin = new HashJoinNode(nextPlanNodeId(), probe, build, JoinOperator.INNER_JOIN,
+ eqConjuncts, Collections.emptyList(), null, null, false);
+ broadcastJoin.setDistributionMode(DistributionMode.BROADCAST);
+ Pair<PlanNode, LocalExchangeType> broadcastOutput = broadcastJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, broadcastOutput.second);
+ Assertions.assertSame(probe, broadcastJoin.getChild(0));
+ Assertions.assertSame(build, broadcastJoin.getChild(1));
+
+ TrackingPlanNode probe2 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode build2 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ HashJoinNode bucketJoin = new HashJoinNode(nextPlanNodeId(), probe2, build2, JoinOperator.INNER_JOIN,
+ eqConjuncts, Collections.emptyList(), null, null, false);
+ bucketJoin.setDistributionMode(DistributionMode.BUCKET_SHUFFLE);
+ Pair<PlanNode, LocalExchangeType> bucketOutput = bucketJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, bucketOutput.second);
+ assertChildLocalExchangeType(bucketJoin, 0, LocalExchangeType.BUCKET_HASH_SHUFFLE);
+ assertChildLocalExchangeType(bucketJoin, 1, LocalExchangeType.BUCKET_HASH_SHUFFLE);
+
+ TrackingScanNode probeScan = new TrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode buildPlan = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ HashJoinNode hashJoin = new HashJoinNode(nextPlanNodeId(), probeScan, buildPlan, JoinOperator.INNER_JOIN,
+ eqConjuncts, Collections.emptyList(), null, null, false);
+ hashJoin.setDistributionMode(DistributionMode.PARTITIONED);
+ Pair<PlanNode, LocalExchangeType> hashOutput = hashJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ // enforceRequire resolves RequireHash to LOCAL_EXECUTION_HASH_SHUFFLE (FE-planned always uses LOCAL)
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, hashOutput.second);
+ assertChildLocalExchangeType(hashJoin, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+ assertChildLocalExchangeType(hashJoin, 1, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+
+ TrackingPlanNode probe3 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode build3 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ HashJoinNode nullAwareJoin = new HashJoinNode(nextPlanNodeId(), probe3, build3,
+ JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN, eqConjuncts, Collections.emptyList(), null, null, false);
+ Pair<PlanNode, LocalExchangeType> nullAwareOutput = nullAwareJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, nullAwareOutput.second);
+ Assertions.assertSame(probe3, nullAwareJoin.getChild(0));
+ Assertions.assertSame(build3, nullAwareJoin.getChild(1));
+
+ SerialTrackingPlanNode serialProbe = new SerialTrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ serialProbe.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(serialProbe.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ TrackingPlanNode nonSerialBuild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ nonSerialBuild.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(nonSerialBuild.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ HashJoinNode serialProbeBroadcast = new HashJoinNode(nextPlanNodeId(), serialProbe, nonSerialBuild,
+ JoinOperator.INNER_JOIN, eqConjuncts, Collections.emptyList(), null, null, false);
+ serialProbeBroadcast.setDistributionMode(DistributionMode.BROADCAST);
+ // BROADCAST serial check uses fragment.useSerialSource() on the HashJoinNode itself
+ serialProbeBroadcast.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(serialProbeBroadcast.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ Pair<PlanNode, LocalExchangeType> serialProbeOutput = serialProbeBroadcast.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, serialProbeOutput.second);
+ assertChildLocalExchangeType(serialProbeBroadcast, 0, LocalExchangeType.PASSTHROUGH);
+ Assertions.assertSame(nonSerialBuild, serialProbeBroadcast.getChild(1));
+
+ TrackingPlanNode nonSerialProbe = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ nonSerialProbe.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(nonSerialProbe.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ SerialTrackingPlanNode serialBuild = new SerialTrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ serialBuild.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(serialBuild.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ HashJoinNode serialBuildBroadcast = new HashJoinNode(nextPlanNodeId(), nonSerialProbe, serialBuild,
+ JoinOperator.INNER_JOIN, eqConjuncts, Collections.emptyList(), null, null, false);
+ serialBuildBroadcast.setDistributionMode(DistributionMode.BROADCAST);
+ serialBuildBroadcast.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(serialBuildBroadcast.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ Pair<PlanNode, LocalExchangeType> serialBuildOutput = serialBuildBroadcast.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, serialBuildOutput.second);
+ Assertions.assertSame(nonSerialProbe, serialBuildBroadcast.getChild(0));
+ assertChildLocalExchangeType(serialBuildBroadcast, 1, LocalExchangeType.PASS_TO_ONE);
+ }
+
+ @Test
+ public void testLocalExchangeNodeIsNotSerializedAsSerialOperator() {
+ SerialTrackingScanNode serialScan = new SerialTrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ LocalExchangeNode localExchangeNode = new LocalExchangeNode(nextPlanNodeId(), serialScan,
+ LocalExchangeType.PASSTHROUGH);
+ localExchangeNode.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(localExchangeNode.fragment.hasSerialScanNode()).thenReturn(true);
+ Mockito.when(localExchangeNode.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+
+ TPlanNode thriftNode = new TPlanNode();
+ localExchangeNode.toThrift(thriftNode);
+
+ Assertions.assertFalse(thriftNode.isIsSerialOperator(),
+ "local exchange source pipeline should not be marked serial in thrift");
+ }
+
+ @Test
+ public void testNestedLoopJoinNodeBranches() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ List<TupleId> tupleIds = Lists.newArrayList(new TupleId(NEXT_ID.getAndIncrement()));
+
+ TrackingPlanNode probe = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode build = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ NestedLoopJoinNode defaultJoin = new NestedLoopJoinNode(nextPlanNodeId(), probe, build, tupleIds,
+ JoinOperator.INNER_JOIN, false);
+ defaultJoin.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(defaultJoin.fragment.useSerialSource(Mockito.any())).thenReturn(false);
+ Pair<PlanNode, LocalExchangeType> defaultOutput = defaultJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.ADAPTIVE_PASSTHROUGH, defaultOutput.second);
+ assertChildLocalExchangeType(defaultJoin, 0, LocalExchangeType.ADAPTIVE_PASSTHROUGH);
+ Assertions.assertSame(build, defaultJoin.getChild(1));
+
+ TrackingScanNode probeScan = new TrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode buildNoop = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ NestedLoopJoinNode serialSourceJoin = new NestedLoopJoinNode(nextPlanNodeId(), probeScan, buildNoop,
+ Lists.newArrayList(new TupleId(NEXT_ID.getAndIncrement())), JoinOperator.INNER_JOIN, false);
+ serialSourceJoin.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(serialSourceJoin.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ Pair<PlanNode, LocalExchangeType> serialOutput = serialSourceJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.ADAPTIVE_PASSTHROUGH, serialOutput.second);
+ assertChildLocalExchangeType(serialSourceJoin, 0, LocalExchangeType.ADAPTIVE_PASSTHROUGH);
+ assertChildLocalExchangeType(serialSourceJoin, 1, LocalExchangeType.BROADCAST);
+
+ // RIGHT_OUTER/FULL_OUTER: probe side must use NOOP (serial processing for unmatched rows).
+ // BE: NestedLoopJoinProbeOperatorX returns NOOP for these join types.
+ TrackingPlanNode probeRight = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode buildRight = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ NestedLoopJoinNode rightOuterJoin = new NestedLoopJoinNode(nextPlanNodeId(), probeRight, buildRight,
+ Lists.newArrayList(new TupleId(NEXT_ID.getAndIncrement())),
+ JoinOperator.RIGHT_OUTER_JOIN, false);
+ rightOuterJoin.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(rightOuterJoin.fragment.useSerialSource(Mockito.any())).thenReturn(false);
+ Pair<PlanNode, LocalExchangeType> rightOuterOutput = rightOuterJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, rightOuterOutput.second);
+ Assertions.assertSame(probeRight, rightOuterJoin.getChild(0));
+
+ TrackingPlanNode probe2 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode build2 = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ NestedLoopJoinNode nullAwareJoin = new NestedLoopJoinNode(nextPlanNodeId(), probe2, build2,
+ Lists.newArrayList(new TupleId(NEXT_ID.getAndIncrement())),
+ JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN, false);
+ nullAwareJoin.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(nullAwareJoin.fragment.useSerialSource(Mockito.any())).thenReturn(false);
+ Pair<PlanNode, LocalExchangeType> nullAwareOutput = nullAwareJoin.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, nullAwareOutput.second);
+ }
+
+ @Test
+ public void testSetOperationAndAssertNumRowsNode() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+
+ UnionNode unionNode = new UnionNode(nextPlanNodeId(), new TupleId(NEXT_ID.getAndIncrement()));
+ TrackingPlanNode unionChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ unionNode.addChild(unionChild);
+ // UnionNode propagates parent hash require to children when parent requires hash.
+ // resolveExchangeType with RequireHash → LOCAL_EXECUTION_HASH_SHUFFLE
+ Pair<PlanNode, LocalExchangeType> unionOutput = unionNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, unionOutput.second);
+ Assertions.assertEquals(LocalExchangeNode.RequireHash.class, unionChild.lastRequire.getClass());
+
+ IntersectNode intersectNode = new IntersectNode(nextPlanNodeId(), new TupleId(NEXT_ID.getAndIncrement()));
+ intersectNode.setColocate(false);
+ TrackingScanNode left = new TrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ TrackingPlanNode right = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ intersectNode.addChild(left);
+ intersectNode.addChild(right);
+ Pair<PlanNode, LocalExchangeType> intersectOutput = intersectNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, intersectOutput.second);
+ assertChildLocalExchangeType(intersectNode, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+ assertChildLocalExchangeType(intersectNode, 1, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+
+ // Colocated ExceptNode with OlapScan children: OlapScan already provides BUCKET_HASH_SHUFFLE,
+ // so requireBucketHash() is satisfied and no LocalExchangeNode is inserted.
+ ExceptNode exceptNode = new ExceptNode(nextPlanNodeId(), new TupleId(NEXT_ID.getAndIncrement()));
+ exceptNode.setColocate(true);
+ FakeOlapScanNode exceptLeft = new FakeOlapScanNode(nextPlanNodeId());
+ FakeOlapScanNode exceptRight = new FakeOlapScanNode(nextPlanNodeId());
+ exceptNode.addChild(exceptLeft);
+ exceptNode.addChild(exceptRight);
+ Pair<PlanNode, LocalExchangeType> exceptOutput = exceptNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, exceptOutput.second);
+ // OlapScan already satisfies requireBucketHash(), so children are passed through unchanged.
+ Assertions.assertSame(exceptLeft, exceptNode.getChild(0));
+ Assertions.assertSame(exceptRight, exceptNode.getChild(1));
+
+ TrackingPlanNode assertChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ AssertNumRowsElement assertElement = Mockito.mock(AssertNumRowsElement.class);
+ Mockito.when(assertElement.getDesiredNumOfRows()).thenReturn(1L);
+ Mockito.when(assertElement.getSubqueryString()).thenReturn("subquery");
+ Mockito.when(assertElement.getAssertion()).thenReturn(AssertNumRowsElement.Assertion.EQ);
+ AssertNumRowsNode assertNode = new AssertNumRowsNode(nextPlanNodeId(), assertChild,
+ assertElement, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ // AssertNumRowsNode.isSerialNode()=true → enforceChild skips exchange.
+ Pair<PlanNode, LocalExchangeType> assertOutput = assertNode.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, assertOutput.second);
+ Assertions.assertSame(assertChild, assertNode.getChild(0));
+ }
+
+ @Test
+ public void testSortNodeBranches() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+ SortInfo sortInfo = mockSortInfo();
+
+ TrackingPlanNode mergeChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ SortNode mergeSort = new SortNode(nextPlanNodeId(), mergeChild, sortInfo, false);
+ mergeSort.setMergeByExchange();
+ Pair<PlanNode, LocalExchangeType> mergeOutput = mergeSort.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, mergeOutput.second);
+ assertChildLocalExchangeType(mergeSort, 0, LocalExchangeType.PASSTHROUGH);
+
+ // Non-merge, non-analytic SortNode: isSerialNode()=true → enforceChild skips exchange.
+ // Output is still PASSTHROUGH (hardcoded for useSerialSource + ScanNode child).
+ SerialTrackingScanNode serialScan = new SerialTrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ SortNode scanSort = new SortNode(nextPlanNodeId(), serialScan, sortInfo, false);
+ scanSort.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(scanSort.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ Pair<PlanNode, LocalExchangeType> scanOutput = scanSort.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ // Non-merge, non-analytic SortNode: isSerialNode()=true, requireChild=noRequire,
+ // outputType=NOOP. enforceRequire shouldSkipLE skips because Sort itself is serial.
+ Assertions.assertEquals(LocalExchangeType.NOOP, scanOutput.second);
+ // SortNode is serial → enforceRequire skips exchange → child unchanged.
+ Assertions.assertSame(serialScan, scanSort.getChild(0));
+
+ // Analytic sort (mergeByexchange=false): sort before analytic with partition + orderBy.
+ // AnalyticEvalNode returns NOOP (non-serial, has partition+order), SortNode enforceChild
+ // inserts LOCAL_EXECUTION_HASH_SHUFFLE (RequireHash → resolveExchangeType → LOCAL).
+ AnalyticEvalNode analyticChild = new AnalyticEvalNode(nextPlanNodeId(),
+ new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP),
+ Collections.emptyList(), Collections.singletonList(Mockito.mock(Expr.class)),
+ Collections.singletonList(new OrderByElement(Mockito.mock(Expr.class), true, true)),
+ null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ analyticChild.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(analyticChild.fragment.useSerialSource(Mockito.any())).thenReturn(false);
+ SortNode analyticSort = new SortNode(nextPlanNodeId(), analyticChild, sortInfo, false);
+ analyticSort.setIsAnalyticSort(true); // Must set for isSerialNode() to return false
+ Pair<PlanNode, LocalExchangeType> analyticOutput = analyticSort.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, analyticOutput.second);
+ assertChildLocalExchangeType(analyticSort, 0, LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE);
+
+ // Outer merge-sort above analytic (mergeByexchange=true): BE SortSink._merge_by_exchange=true → PASSTHROUGH.
+ // Should NOT insert GLOBAL_HASH even though child is AnalyticEvalNode.
+ AnalyticEvalNode analyticChild2 = new AnalyticEvalNode(nextPlanNodeId(),
+ new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP),
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
+ null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ analyticChild2.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(analyticChild2.fragment.useSerialSource(Mockito.any())).thenReturn(false);
+ SortNode mergeAnalyticSort = new SortNode(nextPlanNodeId(), analyticChild2, sortInfo, false);
+ mergeAnalyticSort.setMergeByExchange();
+ Pair<PlanNode, LocalExchangeType> mergeAnalyticOutput = mergeAnalyticSort.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, mergeAnalyticOutput.second);
+ }
+
+ @Test
+ public void testAnalyticEvalNodeBranches() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+
+ TrackingPlanNode noPartitionChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ AnalyticEvalNode noPartition = new AnalyticEvalNode(nextPlanNodeId(), noPartitionChild,
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
+ null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ // No partition → isSerialNode()=true → enforceChild skips exchange.
+ // Output is still PASSTHROUGH (hardcoded for empty partitions).
+ Pair<PlanNode, LocalExchangeType> noPartitionOutput = noPartition.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, noPartitionOutput.second);
+ Assertions.assertSame(noPartitionChild, noPartition.getChild(0));
+
+ // Analytic with partition but no orderBy, non-colocated → noRequire/NOOP.
+ // (Non-colocated analytic relies on parent SortNode to handle distribution.)
+ TrackingScanNode hashChild = new TrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ AnalyticEvalNode hashAnalytic = new AnalyticEvalNode(nextPlanNodeId(), hashChild,
+ Collections.emptyList(), Collections.singletonList(Mockito.mock(Expr.class)),
+ Collections.emptyList(), null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ Pair<PlanNode, LocalExchangeType> hashOutput = hashAnalytic.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, hashOutput.second);
+ // No exchange inserted — child remains unchanged.
+ Assertions.assertSame(hashChild, hashAnalytic.getChild(0));
+
+ SerialTrackingScanNode serialScan = new SerialTrackingScanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
+ AnalyticEvalNode orderedAnalytic = new AnalyticEvalNode(nextPlanNodeId(), serialScan,
+ Collections.emptyList(), Collections.singletonList(Mockito.mock(Expr.class)),
+ Collections.singletonList(new OrderByElement(Mockito.mock(Expr.class), true, true)),
+ null, new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement())));
+ orderedAnalytic.fragment = Mockito.mock(PlanFragment.class);
+ Mockito.when(orderedAnalytic.fragment.useSerialSource(Mockito.any())).thenReturn(true);
+ Pair<PlanNode, LocalExchangeType> orderedOutput = orderedAnalytic.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.PASSTHROUGH, orderedOutput.second);
+ assertChildLocalExchangeType(orderedAnalytic, 0, LocalExchangeType.PASSTHROUGH);
+ }
+
+ @Test
+ public void testExchangeNodeBranches() {
+ PlanTranslatorContext ctx = new PlanTranslatorContext();
+
+ ExchangeNode hashExchange = new ExchangeNode(nextPlanNodeId(),
+ new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP));
+ hashExchange.setPartitionType(TPartitionType.HASH_PARTITIONED);
+ Pair<PlanNode, LocalExchangeType> hashOutput = hashExchange.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE, hashOutput.second);
+
+ ExchangeNode bucketExchange = new ExchangeNode(nextPlanNodeId(),
+ new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP));
+ bucketExchange.setPartitionType(TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED);
+ Pair<PlanNode, LocalExchangeType> bucketOutput = bucketExchange.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.noRequire());
+ Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, bucketOutput.second);
+
+ ExchangeNode noopExchange = new ExchangeNode(nextPlanNodeId(),
+ new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP));
+ noopExchange.setPartitionType(TPartitionType.UNPARTITIONED);
+ Pair<PlanNode, LocalExchangeType> noopOutput = noopExchange.enforceAndDeriveLocalExchange(
+ ctx, null, LocalExchangeTypeRequire.requireHash());
+ Assertions.assertEquals(LocalExchangeType.NOOP, noopOutput.second);
+ }
+
+ private static PlanNodeId nextPlanNodeId() {
+ return new PlanNodeId(NEXT_ID.getAndIncrement());
+ }
+
+ private static void assertChildLocalExchangeType(PlanNode node, int index, LocalExchangeType expectedType) {
+ Assertions.assertTrue(node.getChild(index) instanceof LocalExchangeNode,
+ "expected child " + index + " to be LocalExchangeNode");
+ LocalExchangeNode exchangeNode = (LocalExchangeNode) node.getChild(index);
+ Assertions.assertEquals(expectedType, exchangeNode.getExchangeType());
+ }
+
+ private static SortInfo mockSortInfo() {
+ SortInfo sortInfo = Mockito.mock(SortInfo.class);
+ TupleDescriptor sortTuple = new TupleDescriptor(new TupleId(NEXT_ID.getAndIncrement()));
+ Mockito.when(sortInfo.getOrderingExprs()).thenReturn(Collections.emptyList());
+ Mockito.when(sortInfo.getIsAscOrder()).thenReturn(Collections.emptyList());
+ Mockito.when(sortInfo.getSortTupleDescriptor()).thenReturn(sortTuple);
+ return sortInfo;
+ }
+
+ private static class TrackingPlanNode extends PlanNode {
+ private final LocalExchangeType providedType;
+ private LocalExchangeTypeRequire lastRequire;
+
+ TrackingPlanNode(PlanNodeId id, LocalExchangeType providedType) {
+ super(id, Lists.newArrayList(new TupleId(id.asInt() + 10000)), "TRACKING");
+ this.providedType = providedType;
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ this.lastRequire = parentRequire;
+ return Pair.of(this, providedType);
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ return "";
+ }
+ }
+
+ private static class SerialTrackingPlanNode extends TrackingPlanNode {
+ SerialTrackingPlanNode(PlanNodeId id, LocalExchangeType providedType) {
+ super(id, providedType);
+ }
+
+ @Override
+ public boolean isSerialNode() {
+ return true;
+ }
+ }
+
+ private static class TrackingScanNode extends ScanNode {
+ private final LocalExchangeType providedType;
+ private LocalExchangeTypeRequire lastRequire;
+
+ TrackingScanNode(PlanNodeId id, LocalExchangeType providedType) {
+ super(id, new TupleDescriptor(new TupleId(id.asInt() + 20000)), "TRACKING_SCAN", ScanContext.EMPTY);
+ this.providedType = providedType;
+ }
+
+ @Override
+ public Pair<PlanNode, LocalExchangeType> enforceAndDeriveLocalExchange(
+ PlanTranslatorContext translatorContext, PlanNode parent, LocalExchangeTypeRequire parentRequire) {
+ this.lastRequire = parentRequire;
+ return Pair.of(this, providedType);
+ }
+
+ @Override
+ protected void createScanRangeLocations() throws UserException {
+ }
+
+ @Override
+ public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ return "";
+ }
+ }
+
+ private static class SerialTrackingScanNode extends TrackingScanNode {
+ SerialTrackingScanNode(PlanNodeId id, LocalExchangeType providedType) {
+ super(id, providedType);
+ }
+
+ @Override
+ public boolean isSerialNode() {
+ return true;
+ }
+ }
+
+ private static class FakeOlapScanNode extends OlapScanNode {
+ FakeOlapScanNode(PlanNodeId id) {
+ super(id, mockTupleDescriptor(id), "FAKE_OLAP_SCAN", ScanContext.EMPTY);
+ }
+
+ @Override
+ protected void createScanRangeLocations() throws UserException {
+ }
+
+ @Override
+ public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
+ return Collections.emptyList();
+ }
+
+ private static TupleDescriptor mockTupleDescriptor(PlanNodeId id) {
+ TupleDescriptor desc = Mockito.mock(TupleDescriptor.class);
+ org.apache.doris.catalog.OlapTable table = Mockito.mock(org.apache.doris.catalog.OlapTable.class);
+ Mockito.when(desc.getId()).thenReturn(new TupleId(id.asInt() + 30000));
+ Mockito.when(desc.getTable()).thenReturn(table);
+ Mockito.when(desc.getSlots()).thenReturn(new ArrayList<SlotDescriptor>());
+ Mockito.when(table.getDistributionColumnNames()).thenReturn(Collections.emptySet());
+ return desc;
+ }
+ }
+
+ private static class TestMaterializationNode extends MaterializationNode {
+ TestMaterializationNode(PlanNodeId id, TupleDescriptor desc, PlanNode child) {
+ super(id, desc, child);
+ }
+
+ @Override
+ public void initNodeInfo() {
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalExchangePlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalExchangePlannerTest.java
new file mode 100644
index 00000000000..3d9f522facc
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalExchangePlannerTest.java
@@ -0,0 +1,372 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.planner.AddLocalExchange;
+import org.apache.doris.planner.LocalExchangeNode;
+import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanNode;
+import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.ScanContext;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TScanRangeLocations;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
+public class LocalExchangePlannerTest extends TestWithFeService {
+ @Override
+ protected int backendNum() {
+ return 3;
+ }
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("test");
+ useDatabase("test");
+ createTable("CREATE TABLE test.t1 (k1 INT, k2 INT, v1 INT) "
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 6 "
+ + "PROPERTIES ('replication_num'='1')");
+ createTable("CREATE TABLE test.t2 (k1 INT, k2 INT, v2 INT) "
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 6 "
+ + "PROPERTIES ('replication_num'='1')");
+ }
+
+ @Test
+ public void testAggFromScanUsesLocalExecutionHashShuffle() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(true);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ // With the Bug 7 fix, OlapScanNode returns NOOP (no self-wrapping),
+ // so the parent AggregationNode's requireHash() resolves to
+ // LOCAL_EXECUTION_HASH_SHUFFLE (scan child → local hash).
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, k2, count(*) from test.t1 group by k1, k2");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+ String explain = collectFragmentExplain(planner.getFragments());
+
+ Assertions.assertTrue(types.contains(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE),
+ "expected LOCAL_EXECUTION_HASH_SHUFFLE in plan, actual: " + types);
+ Assertions.assertTrue(explain.contains("LOCAL_EXECUTION_HASH_SHUFFLE"),
+ "expected LOCAL_EXECUTION_HASH_SHUFFLE in explain output, actual explain: " + explain);
+ }
+
+ @Test
+ public void testNonSerialScanKeepsBucketHashDistribution() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(false);
+ connectContext.getSessionVariable().setPipelineTaskNum("1");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, count(*) from test.t1 group by k1 order by k1");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ String explain = collectFragmentExplain(planner.getFragments());
+
+ Assertions.assertFalse(explain.contains("LOCAL_EXECUTION_HASH_SHUFFLE"),
+ "non-serial scan should keep BUCKET_HASH_SHUFFLE output and avoid local hash exchange, explain: "
+ + explain);
+ }
+
+ @Test
+ public void testJoinPlanContainsHashShuffle() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(true);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select a.k1, count(*) "
+ + "from test.t1 a join test.t2 b on a.k1 = b.k1 group by a.k1");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ // With pooling scan and local shuffle planner, hash exchanges should be present
+ boolean hasHashShuffle = types.stream().anyMatch(t -> t.isHashShuffle());
+ Assertions.assertTrue(hasHashShuffle || types.contains(LocalExchangeType.PASSTHROUGH),
+ "expected hash shuffle or passthrough in plan, actual: " + types);
+ }
+
+ @Test
+ public void testNoopLocalExchangeNotInjected() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql("explain distributed plan select * from test.t1 limit 1");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ Assertions.assertFalse(types.contains(LocalExchangeType.NOOP),
+ "NOOP local exchange should not be materialized as a node");
+ }
+
+ @Test
+ public void testHashShuffleHasDistributeExprs() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(true);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ // Use a simple agg query that reliably produces hash local exchange
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, k2, count(*) from test.t1 group by k1, k2");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ List<LocalExchangeNode> localExchanges = collectLocalExchangeNodes(planner.getFragments());
+
+ boolean hasHashShuffleWithExpr = localExchanges.stream().anyMatch(node ->
+ node.getExchangeType().isHashShuffle()
+ && node.getDistributeExprLists() != null
+ && !node.getDistributeExprLists().isEmpty());
+ String exchangeInfo = localExchanges.stream()
+ .map(n -> n.getExchangeType() + "(exprs=" + n.getDistributeExprLists() + ")")
+ .collect(java.util.stream.Collectors.joining(", "));
+ Assertions.assertTrue(hasHashShuffleWithExpr,
+ "expected at least one hash local exchange with distribute exprs, found: " + exchangeInfo);
+ }
+
+ @Test
+ public void testRequireHashSatisfyAllHashShuffleTypes() {
+ LocalExchangeNode.LocalExchangeTypeRequire requireHash = LocalExchangeNode.LocalExchangeTypeRequire.requireHash();
+ Assertions.assertTrue(requireHash.satisfy(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE));
+ Assertions.assertTrue(requireHash.satisfy(LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE));
+ Assertions.assertTrue(requireHash.satisfy(LocalExchangeType.BUCKET_HASH_SHUFFLE));
+ Assertions.assertFalse(requireHash.satisfy(LocalExchangeType.PASSTHROUGH));
+ }
+
+ @Test
+ public void testSetOperationUnderAggHasHashShuffle() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(false);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, count(*) from ("
+ + "select k1 from test.t1 union all select k1 from test.t2"
+ + ") u group by k1");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ // With non-pooling scan and colocated bucket distribution, local exchanges may
+ // not be inserted. Verify plan at least doesn't crash and contains valid exchange types.
+ boolean hasLocalExchange = !types.isEmpty();
+ // If local exchanges are present, they should include hash shuffle types
+ if (hasLocalExchange) {
+ boolean hasHashShuffle = types.stream().anyMatch(t -> t.isHashShuffle());
+ Assertions.assertTrue(hasHashShuffle,
+ "expected hash shuffle in set-operation plan when exchanges present, actual: " + types);
+ }
+ }
+
+ @Test
+ public void testAnalyticPlanContainsPassthroughAndLocalHashShuffle() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(true);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, k2, row_number() over(partition by k1 order by k2) "
+ + "from test.t1 order by k1, k2");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ // Analytic plan: mergeByExchange sort inserts PASSTHROUGH.
+ // With pooling scan (ignore_storage_data_distribution=true), hash or passthrough exchanges expected.
+ Assertions.assertTrue(types.contains(LocalExchangeType.PASSTHROUGH),
+ "expected PASSTHROUGH in analytic plan, actual: " + types);
+ }
+
+ @Test
+ public void testGroupingSetsPlanContainsHashShuffle() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(false);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select k1, k2, sum(v1) from test.t1 "
+ + "group by grouping sets((k1), (k1, k2))");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ Assertions.assertFalse(types.contains(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE),
+ "grouping-sets plan should not force local execution hash shuffle when scan keeps bucket"
+ + " distribution, actual: " + types);
+ }
+
+ @Test
+ public void testLocalAndGlobalExecutionHashShufflePreferType() {
+ PlanTranslatorContext translatorContext = new PlanTranslatorContext();
+ LocalExchangeNode.LocalExchangeTypeRequire requireHash = LocalExchangeNode.LocalExchangeTypeRequire.requireHash();
+ LocalExchangeNode.LocalExchangeTypeRequire requireBucketHash
+ = LocalExchangeNode.LocalExchangeTypeRequire.requireBucketHash();
+ LocalExchangeNode.LocalExchangeTypeRequire requireGlobalHash
+ = LocalExchangeNode.LocalExchangeTypeRequire.requireGlobalExecutionHash();
+
+ LocalExchangeType localType = AddLocalExchange.resolveExchangeType(
+ requireHash, translatorContext, null, new MockScanNode(new PlanNodeId(1001)));
+ LocalExchangeType globalType = AddLocalExchange.resolveExchangeType(
+ requireHash, translatorContext, null, new MockPlanNode(new PlanNodeId(1002)));
+ // Explicit GLOBAL_EXECUTION_HASH_SHUFFLE must NOT be degraded, even on a scan path.
+ // If it appears on a scan path, the plan is wrong — not something resolveExchangeType should fix.
+ LocalExchangeType explicitGlobalOnScanType = AddLocalExchange.resolveExchangeType(
+ requireGlobalHash, translatorContext, null, new MockScanNode(new PlanNodeId(1003)));
+
+ // shouldUseLocalExecutionHash always returns true → RequireHash always resolves to LOCAL
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, localType);
+ Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, globalType);
+ // Explicit GLOBAL (RequireSpecific) must NOT be degraded.
+ Assertions.assertEquals(LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE, explicitGlobalOnScanType);
+ Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, requireBucketHash.preferType());
+ }
+
+ @Test
+ public void testMixedPlanWithPoolingScan() throws Exception {
+ connectContext.getSessionVariable().setEnableLocalShufflePlanner(true);
+ connectContext.getSessionVariable().setEnableLocalShuffle(true);
+ connectContext.getSessionVariable().setEnableNereidsDistributePlanner(true);
+ connectContext.getSessionVariable().setIgnoreStorageDataDistribution(true);
+ connectContext.getSessionVariable().setPipelineTaskNum("4");
+ connectContext.getSessionVariable().setForceToLocalShuffle(false);
+
+ StmtExecutor executor = executeNereidsSql(
+ "explain distributed plan select u.k1, count(*) from ("
+ + "select k1, k2 from test.t1 group by grouping sets((k1), (k1, k2))"
+ + ") u join test.t2 b on u.k1 = b.k1 group by u.k1");
+ NereidsPlanner planner = (NereidsPlanner) executor.planner();
+ EnumSet<LocalExchangeType> types = collectLocalExchangeTypes(planner.getFragments());
+
+ // With pooling scan, local exchanges should be present
+ Assertions.assertFalse(types.isEmpty(),
+ "expected local exchanges in mixed plan with pooling scan, actual: " + types);
+ }
+
+ private EnumSet<LocalExchangeType> collectLocalExchangeTypes(List<PlanFragment> fragments) {
+ EnumSet<LocalExchangeType> types = EnumSet.noneOf(LocalExchangeType.class);
+ for (PlanFragment fragment : fragments) {
+ collect(fragment.getPlanRoot(), types);
+ }
+ return types;
+ }
+
+ private List<LocalExchangeNode> collectLocalExchangeNodes(List<PlanFragment> fragments) {
+ List<LocalExchangeNode> nodes = new ArrayList<>();
+ for (PlanFragment fragment : fragments) {
+ collectLocalExchangeNode(fragment.getPlanRoot(), nodes);
+ }
+ return nodes;
+ }
+
+ private String collectFragmentExplain(List<PlanFragment> fragments) {
+ StringBuilder explain = new StringBuilder();
+ for (PlanFragment fragment : fragments) {
+ explain.append(fragment.getExplainString(TExplainLevel.NORMAL));
+ }
+ return explain.toString();
+ }
+
+ private void collect(PlanNode node, EnumSet<LocalExchangeType> types) {
+ if (node instanceof LocalExchangeNode) {
+ types.add(((LocalExchangeNode) node).getExchangeType());
+ }
+ for (PlanNode child : node.getChildren()) {
+ collect(child, types);
+ }
+ }
+
+ private void collectLocalExchangeNode(PlanNode node, List<LocalExchangeNode> nodes) {
+ if (node instanceof LocalExchangeNode) {
+ nodes.add((LocalExchangeNode) node);
+ }
+ for (PlanNode child : node.getChildren()) {
+ collectLocalExchangeNode(child, nodes);
+ }
+ }
+
+ private static class MockPlanNode extends PlanNode {
+ MockPlanNode(PlanNodeId id) {
+ super(id, "MOCK-PLAN");
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ return "";
+ }
+ }
+
+ private static class MockScanNode extends ScanNode {
+ MockScanNode(PlanNodeId id) {
+ super(id, new TupleDescriptor(new TupleId(id.asInt())), "MOCK-SCAN", ScanContext.EMPTY);
+ }
+
+ @Override
+ protected void createScanRangeLocations() throws UserException {
+ }
+
+ @Override
+ public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
+ return java.util.Collections.emptyList();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ return "";
+ }
+ }
+}
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index eca17033647..03ce2c12724 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -471,7 +471,6 @@ struct TQueryOptions {
// session variable `spill_repartition_max_depth` in FE. Default is 8.
209: optional i32 spill_repartition_max_depth = 8
-
210: optional double max_scan_mem_ratio = 0.3;
211: optional bool enable_adaptive_scan = false;
@@ -489,6 +488,10 @@ struct TQueryOptions {
// Default 8MB. Sent by FE session variable preferred_block_size_bytes.
218: optional i64 preferred_block_size_bytes = 8388608
+ // enable plan local exchange node in fe
+ 219: optional bool enable_local_shuffle_planner;
+
+
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
// In read path, read from file cache or remote storage when execute query.
diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift
index de6ba8127af..bf23f418eaf 100644
--- a/gensrc/thrift/Partitions.thrift
+++ b/gensrc/thrift/Partitions.thrift
@@ -55,6 +55,40 @@ enum TPartitionType {
MERGE_PARTITIONED = 9
}
+enum TLocalPartitionType {
+ NOOP = 0,
+ // used to resume the global hash distribution because other distribution break the global hash distribution,
+ // such as PASSTHROUGH. and then JoinNode can shuffle data by the same hash distribution.
+ //
+ // for example: look here, need resume to GLOBAL_EXECUTION_HASH_SHUFFLE
+ // ↓
+ // Node -> LocalExchangeNode(PASSTHROUGH) → JoinNode → LocalExchangeNode(GLOBAL_EXECUTION_HASH_SHUFFLE) → JoinNode
+ // ExchangeNode(BROADCAST) ↗ ↑
+ // ExchangeNode(GLOBAL_EXECUTION_HASH_SHUFFLE)
+ GLOBAL_EXECUTION_HASH_SHUFFLE = 1,
+ // used to rebalance data for rebalance data and add parallelism
+ //
+ // for example: look here, need use LOCAL_EXECUTION_HASH_SHUFFLE to rebalance data
+ // ↓
+ // Scan(hash(id)) -> LocalExchangeNode(LOCAL_EXECUTION_HASH_SHUFFLE(id, name)) → AggregationNode(group by(id,name))
+ //
+ // the LOCAL_EXECUTION_HASH_SHUFFLE is necessary because the hash distribution of scan node is based on id,
+ // but the hash distribution of aggregation node is based on id and name, so we need to rebalance data by both
+ // id and name to make sure the data with same id and name can be sent to the same instance of aggregation node.
+ // and we can not use GLOBAL_EXECUTION_HASH_SHUFFLE(id, name) here, because
+ // `TPipelineFragmentParams.shuffle_idx_to_instance_idx` is used to mapping partial global instance index to local
+ // instance index, and discard the other backend's instance index, the data not belong to the local instance will be
+ // discarded, which cause data loss.
+ LOCAL_EXECUTION_HASH_SHUFFLE = 2,
+ BUCKET_HASH_SHUFFLE = 3,
+ // round-robin partition, used to rebalance data for rebalance data and add parallelism
+ PASSTHROUGH = 4,
+ ADAPTIVE_PASSTHROUGH = 5,
+ BROADCAST = 6,
+ PASS_TO_ONE = 7,
+ LOCAL_MERGE_SORT = 8
+}
+
enum TDistributionType {
UNPARTITIONED = 0,
@@ -119,3 +153,5 @@ struct TDataPartition {
3: optional list<TRangePartition> partition_infos
4: optional TMergePartitionInfo merge_partition_info
}
+
+
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 843087ff45a..a28729a3a0c 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -63,7 +63,8 @@ enum TPlanNodeType {
MATERIALIZATION_NODE = 34,
REC_CTE_NODE = 35,
REC_CTE_SCAN_NODE = 36,
- BUCKETED_AGGREGATION_NODE = 37
+ BUCKETED_AGGREGATION_NODE = 37,
+ LOCAL_EXCHANGE_NODE = 38
}
struct TKeyRange {
@@ -1361,6 +1362,24 @@ struct TExchangeNode {
4: optional Partitions.TPartitionType partition_type
}
+struct TLocalExchangeNode {
+ 1: required Partitions.TLocalPartitionType partition_type
+ // when partition_type in (GLOBAL_EXECUTION_HASH_SHUFFLE, LOCAL_EXECUTION_HASH_SHUFFLE, BUCKET_HASH_SHUFFLE),
+ // the distribute_expr_lists is not null, and the legacy `TPlanNode.distribute_expr_lists` is deprecated
+ //
+ // the hash computation:
+ // 1. for BUCKET_HASH_SHUFFLE, use distribution_exprs to compute hash value and mod by
+ // `TPipelineFragmentParams.num_buckets`, and mapping bucket index to local instance id by
+ // `TPipelineFragmentParams.bucket_seq_to_instance_idx`
+ // 2. for LOCAL_EXECUTION_HASH_SHUFFLE, use distribution_exprs to compute hash value and mod by
+ // `TPipelineFragmentParams.local_params.size`, and backend will mapping instance index to local instance
+ // by `i -> i`, for example: 1 -> 1, 2 -> 2, ...
+ // 3. for GLOBAL_EXECUTION_HASH_SHUFFLE, use distribution_exprs to compute hash value and mod by
+ // `TPipelineFragmentParams.total_instances`, and mapping global instance index to local instance by
+ // `TPipelineFragmentParams.shuffle_idx_to_instance_idx`
+ 2: optional list<Exprs.TExpr> distribute_expr_lists
+}
+
struct TOlapRewriteNode {
1: required list<Exprs.TExpr> columns
2: required list<Types.TColumnType> column_types
@@ -1578,6 +1597,7 @@ struct TPlanNode {
51: optional bool is_serial_operator
52: optional TRecCTEScanNode rec_cte_scan_node
53: optional TBucketedAggregationNode bucketed_agg_node
+ 54: optional TLocalExchangeNode local_exchange_node
// projections is final projections, which means projecting into results and materializing them into the output block.
101: optional list<Exprs.TExpr> projections
diff --git a/regression-test/data/query_p0/join/test_multilevel_join_agg_local_shuffle.out b/regression-test/data/query_p0/join/test_multilevel_join_agg_local_shuffle.out
new file mode 100644
index 00000000000..8ccf2921dfa
--- /dev/null
+++ b/regression-test/data/query_p0/join/test_multilevel_join_agg_local_shuffle.out
@@ -0,0 +1,2865 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !bucket_shuffle_broadcast_agg_stage_1_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_shuffle_broadcast_agg_stage_1_result_on --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_agg_stage_1_result_off --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_agg_stage_2_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_shuffle_broadcast_agg_stage_2_result_on --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_agg_stage_2_result_off --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_agg_stage_3_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_shuffle_broadcast_agg_stage_3_result_on --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_agg_stage_3_result_off --
+1 52 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_1_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d2.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !shuffle_broadcast_broadcast_agg_stage_1_result_on --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_1_result_off --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_2_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d2.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !shuffle_broadcast_broadcast_agg_stage_2_result_on --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_2_result_off --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_3_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d2.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = d1.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !shuffle_broadcast_broadcast_agg_stage_3_result_on --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !shuffle_broadcast_broadcast_agg_stage_3_result_off --
+1 10 62 10
+2 5 54 20
+3 5 38 30
+4 6 49 40
+
+-- !bucket_broadcast_shuffle_agg_stage_1_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_broadcast_shuffle_agg_stage_1_result_on --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_broadcast_shuffle_agg_stage_1_result_off --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_broadcast_shuffle_agg_stage_2_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_broadcast_shuffle_agg_stage_2_result_on --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_broadcast_shuffle_agg_stage_2_result_off --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_broadcast_shuffle_agg_stage_3_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = c1.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ab.k1 = d.k1)) otherCondition=()
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_broadcast_shuffle_agg_stage_3_result_on --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_broadcast_shuffle_agg_stage_3_result_off --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !alternating_bucket_bucket_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+-- !alternating_bucket_bucket_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_bucket_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_bucket_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_bucket_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_bucket_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_bucket_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_bucket_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_bucket_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_shuffle_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_shuffle_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_shuffle_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_shuffle_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_broadcast_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_broadcast_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_bucket_broadcast_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_bucket_broadcast_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_bucket_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_bucket_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_bucket_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_bucket_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_shuffle_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_shuffle_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_shuffle_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_shuffle_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_broadcast_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_broadcast_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_shuffle_broadcast_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_shuffle_broadcast_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_bucket_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_bucket_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_bucket_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_bucket_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_shuffle_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_shuffle_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_shuffle_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_shuffle_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_broadcast_bucket_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_bucket_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_shuffle_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1 [shuffle]_2
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_broadcast_shuffle_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_shuffle_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((s2.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((expr_cast(k1 as BIGINT) = c.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !alternating_broadcast_broadcast_broadcast_result_on --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !alternating_broadcast_broadcast_broadcast_result_off --
+1 63 16 11
+2 16 24 22
+3 16 35 33
+4 19 46 44
+
+-- !window_union_all_bucket_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalUnion
+------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_union_all_bucket_row_number_result_on --
+1 52 4 40
+2 21 3 60
+3 16 2 60
+4 20 2 80
+
+-- !window_union_all_bucket_row_number_result_off --
+1 52 4 40
+2 21 3 60
+3 16 2 60
+4 20 2 80
+
+-- !window_union_all_bucket_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalUnion
+------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_union_all_bucket_window_sum_result_on --
+1 10 13 40
+2 6 7 60
+3 3 8 60
+4 3 10 80
+
+-- !window_union_all_bucket_window_sum_result_off --
+1 10 13 40
+2 6 7 60
+3 3 8 60
+4 3 10 80
+
+-- !window_union_all_shuffle_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalUnion
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_all_shuffle_row_number_result_on --
+1 208 8 44
+2 21 3 21
+3 16 2 16
+4 20 2 18
+
+-- !window_union_all_shuffle_row_number_result_off --
+1 208 8 44
+2 21 3 21
+3 16 2 16
+4 20 2 18
+
+-- !window_union_all_shuffle_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalUnion
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_all_shuffle_window_sum_result_on --
+1 36 26 44
+2 6 7 21
+3 3 8 16
+4 3 10 18
+
+-- !window_union_all_shuffle_window_sum_result_off --
+1 36 26 44
+2 6 7 21
+3 3 8 16
+4 3 10 18
+
+-- !window_union_all_broadcast_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalUnion
+--------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_all_broadcast_row_number_result_on --
+1 52 4 40
+2 21 3 60
+3 16 2 60
+4 20 2 80
+
+-- !window_union_all_broadcast_row_number_result_off --
+1 52 4 40
+2 21 3 60
+3 16 2 60
+4 20 2 80
+
+-- !window_union_all_broadcast_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalUnion
+--------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_all_broadcast_window_sum_result_on --
+1 10 13 40
+2 6 7 60
+3 3 8 60
+4 3 10 80
+
+-- !window_union_all_broadcast_window_sum_result_off --
+1 10 13 40
+2 6 7 60
+3 3 8 60
+4 3 10 80
+
+-- !window_except_bucket_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalExcept
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_except_bucket_row_number_result_on --
+1 10 2 20
+2 10 2 40
+3 5 1 30
+4 6 1 40
+
+-- !window_except_bucket_row_number_result_off --
+1 10 2 20
+2 10 2 40
+3 5 1 30
+4 6 1 40
+
+-- !window_except_bucket_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalExcept
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_except_bucket_window_sum_result_on --
+1 3 5 20
+2 3 5 40
+3 1 5 30
+4 1 6 40
+
+-- !window_except_bucket_window_sum_result_off --
+1 3 5 20
+2 3 5 40
+3 1 5 30
+4 1 6 40
+
+-- !window_except_shuffle_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalExcept
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_except_shuffle_row_number_result_on --
+1 40 4 22
+2 10 2 14
+3 5 1 8
+4 6 1 9
+
+-- !window_except_shuffle_row_number_result_off --
+1 40 4 22
+2 10 2 14
+3 5 1 8
+4 6 1 9
+
+-- !window_except_shuffle_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalExcept
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_except_shuffle_window_sum_result_on --
+1 10 10 22
+2 3 5 14
+3 1 5 8
+4 1 6 9
+
+-- !window_except_shuffle_window_sum_result_off --
+1 10 10 22
+2 3 5 14
+3 1 5 8
+4 1 6 9
+
+-- !window_except_broadcast_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalExcept
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_except_broadcast_row_number_result_on --
+1 10 2 20
+2 10 2 40
+3 5 1 30
+4 6 1 40
+
+-- !window_except_broadcast_row_number_result_off --
+1 10 2 20
+2 10 2 40
+3 5 1 30
+4 6 1 40
+
+-- !window_except_broadcast_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalExcept
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_except_broadcast_window_sum_result_on --
+1 3 5 20
+2 3 5 40
+3 1 5 30
+4 1 6 40
+
+-- !window_except_broadcast_window_sum_result_off --
+1 3 5 20
+2 3 5 40
+3 1 5 30
+4 1 6 40
+
+-- !window_intersect_bucket_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalIntersect
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_intersect_bucket_row_number_result_on --
+
+-- !window_intersect_bucket_row_number_result_off --
+
+-- !window_intersect_bucket_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+----------------------PhysicalIntersect
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+-- !window_intersect_bucket_window_sum_result_on --
+
+-- !window_intersect_bucket_window_sum_result_off --
+
+-- !window_intersect_shuffle_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalIntersect
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_intersect_shuffle_row_number_result_on --
+
+-- !window_intersect_shuffle_row_number_result_off --
+
+-- !window_intersect_shuffle_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = rhs.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalIntersect
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_intersect_shuffle_window_sum_result_on --
+
+-- !window_intersect_shuffle_window_sum_result_off --
+
+-- !window_intersect_broadcast_row_number_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalIntersect
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_intersect_broadcast_row_number_result_on --
+
+-- !window_intersect_broadcast_row_number_result_off --
+
+-- !window_intersect_broadcast_window_sum_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((mid_q.k1 = rhs.k1)) otherCondition=()
+------------------------PhysicalIntersect
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(rhs)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_intersect_broadcast_window_sum_result_on --
+
+-- !window_intersect_broadcast_window_sum_result_off --
+
+-- !bucket_broadcast_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !bucket_broadcast_agg_result_on --
+1 26 10
+2 9 20
+3 8 30
+4 10 40
+
+-- !bucket_broadcast_agg_result_off --
+1 26 10
+2 9 20
+3 8 30
+4 10 40
+
+-- !partitioned_broadcast_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !partitioned_broadcast_agg_result_on --
+1 32 10
+2 19 20
+3 13 30
+4 15 40
+
+-- !partitioned_broadcast_agg_result_off --
+1 32 10
+2 19 20
+3 13 30
+4 15 40
+
+-- !bucket_partitioned_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecHash]
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !bucket_partitioned_agg_result_on --
+1 52 44
+2 9 14
+3 8 8
+4 10 9
+
+-- !bucket_partitioned_agg_result_off --
+1 52 44
+2 9 14
+3 8 8
+4 10 9
+
+-- !all_three_multilevel_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((y.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !all_three_multilevel_agg_result_on --
+1 96 10
+2 23 20
+3 16 30
+4 19 40
+
+-- !all_three_multilevel_agg_result_off --
+1 96 10
+2 23 20
+3 16 30
+4 19 40
+
+-- !agg_join_agg_mix_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((l.k1 = d.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = r.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !agg_join_agg_mix_result_on --
+1 10 11 10
+2 5 7 20
+3 5 8 30
+4 6 9 40
+
+-- !agg_join_agg_mix_result_off --
+1 10 11 10
+2 5 7 20
+3 5 8 30
+4 6 9 40
+
+-- !double_broadcast_after_bucket_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d2.k1)) otherCondition=()
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d1.k1)) otherCondition=()
+------------------hashAgg[GLOBAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !double_broadcast_after_bucket_result_on --
+1 26 20
+2 9 40
+3 8 60
+4 10 80
+
+-- !double_broadcast_after_bucket_result_off --
+1 26 20
+2 9 40
+3 8 60
+4 10 80
+
+-- !partitioned_join_between_two_aggs_then_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((m.k1 = d.k1)) otherCondition=()
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = r.k1)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalProject
+----------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !partitioned_join_between_two_aggs_then_broadcast_result_on --
+1 16 10
+2 12 20
+3 13 30
+4 15 40
+
+-- !partitioned_join_between_two_aggs_then_broadcast_result_off --
+1 16 10
+2 12 20
+3 13 30
+4 15 40
+
+-- !bucket_shuffle_broadcast_two_stage_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((y.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !bucket_shuffle_broadcast_two_stage_agg_result_on --
+1 52 44 10
+2 9 14 20
+3 8 8 30
+4 10 9 40
+
+-- !bucket_shuffle_broadcast_two_stage_agg_result_off --
+1 52 44 10
+2 9 14 20
+3 8 8 30
+4 10 9 40
+
+-- !left_join_null_preserving_with_multilevel_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[LEFT_OUTER_JOIN bucketShuffle] hashCondition=((a.k1 = t.k1)) otherCondition=()
+--------------PhysicalProject
+----------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !left_join_null_preserving_with_multilevel_agg_result_on --
+1 5 44
+2 5 14
+3 5 8
+4 6 9
+
+-- !left_join_null_preserving_with_multilevel_agg_result_off --
+1 5 44
+2 5 14
+3 5 8
+4 6 9
+
+-- !seven_layer_bucket_shuffle_broadcast_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------hashAgg[GLOBAL]
+------------PhysicalProject
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((m.k1 = d2.k1)) otherCondition=()
+----------------hashAgg[GLOBAL]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((l.k1 = d1.k1)) otherCondition=()
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = y.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------hashAgg[GLOBAL]
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------------------------hashAgg[GLOBAL]
+----------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------hashAgg[LOCAL]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+----------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !seven_layer_bucket_shuffle_broadcast_result_on --
+1 37 10
+2 16 20
+3 16 30
+4 19 40
+
+-- !seven_layer_bucket_shuffle_broadcast_result_off --
+1 37 10
+2 16 20
+3 16 30
+4 19 40
+
+-- !eight_layer_mixed_join_agg_chain_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((n.k1 = d2.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((m.k1 = d1.k1)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !eight_layer_mixed_join_agg_chain_result_on --
+1 52 44 20
+2 9 14 40
+3 8 8 60
+4 10 9 80
+
+-- !eight_layer_mixed_join_agg_chain_result_off --
+1 52 44 20
+2 9 14 40
+3 8 8 60
+4 10 9 80
+
+-- !seven_layer_left_join_mix_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashAgg[LOCAL]
+--------------------PhysicalProject
+----------------------hashJoin[LEFT_OUTER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = r.k1)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------hashAgg[GLOBAL]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------hashAgg[LOCAL]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !seven_layer_left_join_mix_result_on --
+1 10 11 10
+2 5 7 20
+3 5 8 30
+4 6 9 40
+
+-- !seven_layer_left_join_mix_result_off --
+1 10 11 10
+2 5 7 20
+3 5 8 30
+4 6 9 40
+
+-- !broadcast_shuffle_broadcast_nested_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((y.k1 = d2.k1)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((x.k1 = d1.k1)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((expr_cast(k1 as BIGINT) = r.k1)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------hashAgg[GLOBAL]
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((a.k1 = b.k1)) otherCondition=()
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+------------------------------hashAgg[GLOBAL]
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------hashAgg[LOCAL]
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+--------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d1)]
+--------------PhysicalProject
+----------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d2)]
+
+Hint log:
+Used: [shuffle]_1 [broadcast]_2
+UnUsed:
+SyntaxError:
+
+-- !broadcast_shuffle_broadcast_nested_agg_result_on --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !broadcast_shuffle_broadcast_nested_agg_result_off --
+1 26 11 10
+2 9 7 20
+3 8 8 30
+4 10 9 40
+
+-- !window_union_join_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN broadcast] hashCondition=((w.k1 = d.k1)) otherCondition=()
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalUnion
+----------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a(a)]
+----------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b(b)]
+--------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_join_agg_result_on --
+1 13 14
+2 7 23
+3 8 32
+4 10 42
+
+-- !window_union_join_agg_result_off --
+1 13 14
+2 7 23
+3 8 32
+4 10 42
+
+-- !window_except_join_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalQuickSort[LOCAL_SORT]
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((e.k1 = d.k1)) otherCondition=()
+------------------------PhysicalExcept
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalProject
+------------------------------filter((test_multilevel_join_agg_local_shuffle_c.k4 = 1004))
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_except_join_agg_result_on --
+1 1 10
+2 1 20
+3 1 30
+
+-- !window_except_join_agg_result_off --
+1 1 10
+2 1 20
+3 1 30
+
+-- !window_intersect_shuffle_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalProject
+------------PhysicalWindow
+--------------PhysicalWindow
+----------------PhysicalQuickSort[LOCAL_SORT]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------PhysicalProject
+----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------------PhysicalProject
+--------------------------PhysicalIntersect
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_intersect_shuffle_agg_result_on --
+1 22 2
+2 7 1
+3 8 1
+4 9 1
+
+-- !window_intersect_shuffle_agg_result_off --
+1 22 2
+2 7 1
+3 8 1
+4 9 1
+
+-- !window_union_except_broadcast_agg_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------hashAgg[GLOBAL]
+------------PhysicalProject
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((s.k1 = d.k1)) otherCondition=()
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalExcept
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalUnion
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+----------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------filter((test_multilevel_join_agg_local_shuffle_d.k1 = 4))
+------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d]
+----------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_d(d)]
+
+Hint log:
+Used: [broadcast]_1
+UnUsed:
+SyntaxError:
+
+-- !window_union_except_broadcast_agg_result_on --
+1 13 4 10
+2 7 3 20
+3 8 2 30
+4 10 2 40
+
+-- !window_union_except_broadcast_agg_result_off --
+1 13 4 10
+2 7 3 20
+3 8 2 30
+4 10 2 40
+
+-- !window_setop_join_agg_chain_shape_on --
+PhysicalResultSink
+--PhysicalQuickSort[MERGE_SORT]
+----PhysicalDistribute[DistributionSpecGather]
+------PhysicalQuickSort[LOCAL_SORT]
+--------hashAgg[GLOBAL]
+----------PhysicalDistribute[DistributionSpecHash]
+------------hashAgg[LOCAL]
+--------------PhysicalProject
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(k1 as BIGINT) = expr_(cast(k4 as BIGINT) - 1000))) otherCondition=()
+------------------PhysicalProject
+--------------------PhysicalWindow
+----------------------PhysicalWindow
+------------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------PhysicalUnion
+------------------------------PhysicalDistribute[DistributionSpecExecutionAny]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_a]
+------------------------------PhysicalIntersect
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_b]
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------PhysicalProject
+------------------------------------filter((test_multilevel_join_agg_local_shuffle_c.k4 >= 1001))
+--------------------------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c]
+------------------PhysicalProject
+--------------------PhysicalOlapScan[test_multilevel_join_agg_local_shuffle_c(c)]
+
+Hint log:
+Used: [shuffle]_1
+UnUsed:
+SyntaxError:
+
+-- !window_setop_join_agg_chain_result_on --
+1 20 2 22
+2 10 2 14
+3 5 1 8
+4 6 1 9
+
+-- !window_setop_join_agg_chain_result_off --
+1 20 2 22
+2 10 2 14
+3 5 1 8
+4 6 1 9
+
diff --git a/regression-test/plugins/plugin_profile_plan_tree.groovy b/regression-test/plugins/plugin_profile_plan_tree.groovy
new file mode 100644
index 00000000000..ee86810ca79
--- /dev/null
+++ b/regression-test/plugins/plugin_profile_plan_tree.groovy
@@ -0,0 +1,274 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.Suite
+
+/**
+ * Parses the MergedProfile section of a Doris query profile and returns
+ * the pipeline operator tree in an explain-like format.
+ *
+ * Profile text structure (MergedProfile section):
+ * MergedProfile:
+ * Fragments:
+ * Fragment N: ← indent level A
+ * Pipeline N(instance_num=M): ← indent level B (B = A + 2)
+ * - WaitWorkerTime: ... ← pipeline stats (skip)
+ * OPERATOR_NAME(...): ← indent level C (C = B + 2)
+ * - PlanInfo ← plan info marker
+ * - key: value ← plan info item
+ * CommonCounters: ← counter section start (stop reading plan info)
+ * - ...
+ *
+ * Usage in a test suite:
+ * // From raw profile text:
+ * def tree = profile_plan_tree(profileText)
+ * logger.info(tree)
+ *
+ * // From a query_id (fetches profile via HTTP):
+ * def tree = profile_plan_tree_from_id(queryId)
+ * logger.info(tree)
+ */
+
+// ---------------------------------------------------------------------------
+// profile_plan_tree(profileText) → formatted String
+// ---------------------------------------------------------------------------
+Suite.metaClass.profile_plan_tree = { String profileText ->
+
+ // ── Locate MergedProfile section ──────────────────────────────────────
+ def mergedStart = profileText.indexOf("MergedProfile:")
+ if (mergedStart == -1) {
+ return "(no MergedProfile section found in profile)"
+ }
+ def mergedSection = profileText.substring(mergedStart)
+
+ // ── Line-by-line state machine ─────────────────────────────────────────
+ // We detect indent levels dynamically: the first Fragment line sets the
+ // baseline; Pipeline and Operator levels follow by relative indentation.
+ int fragmentIndent = -1 // indent of "Fragment N:" lines
+ int pipelineIndent = -1 // indent of "Pipeline N(...):" lines
+ int operatorIndent = -1 // indent of "OPERATOR_NAME(...):" lines
+ int planInfoIndent = -1 // indent of "- PlanInfo" lines
+ int planItemIndent = -1 // indent of "- key: value" plan info items
+ int counterIndent = -1 // indent of "CommonCounters:" / "CustomCounters:"
+
+ // Parsed tree: list of fragments, each with pipelines, each with operators
+ def fragments = [] // [{name, pipelines:[{name,instanceNum,ops:[{name,planInfo:[]}]}]}]
+
+ def curFragName = null
+ def curPipeName = null
+ def curOpName = null
+ def curPlanInfo = [] // accumulated plan-info lines for current operator
+ def inPlanInfo = false
+ def curPipeOps = [] // operators in current pipeline
+ def curFragPipes = [] // pipelines in current fragment
+
+ def flushOp = {
+ if (curOpName != null) {
+ curPipeOps << [name: curOpName, planInfo: new ArrayList(curPlanInfo)]
+ curOpName = null
+ curPlanInfo.clear()
+ inPlanInfo = false
+ }
+ }
+
+ def flushPipeline = {
+ flushOp()
+ if (curPipeName != null) {
+ curFragPipes << [name: curPipeName, ops: new ArrayList(curPipeOps)]
+ curPipeName = null
+ curPipeOps.clear()
+ }
+ }
+
+ def flushFragment = {
+ flushPipeline()
+ if (curFragName != null) {
+ fragments << [name: curFragName, pipelines: new ArrayList(curFragPipes)]
+ curFragName = null
+ curFragPipes.clear()
+ }
+ }
+
+ for (def rawLine : mergedSection.split("\n")) {
+ // Count leading spaces
+ int spaces = 0
+ for (char c : rawLine.toCharArray()) {
+ if (c == ' ') spaces++
+ else break
+ }
+ def content = rawLine.trim()
+ if (content.isEmpty()) continue
+
+ // ── Fragment ───────────────────────────────────────────────────────
+ if (content =~ /^Fragment \d+:$/) {
+ if (fragmentIndent == -1) fragmentIndent = spaces
+ if (spaces == fragmentIndent) {
+ flushFragment()
+ curFragName = content[0..-2] // strip trailing ':'
+ // Reset derived indent markers when a new fragment begins
+ pipelineIndent = -1
+ operatorIndent = -1
+ planInfoIndent = -1
+ planItemIndent = -1
+ counterIndent = -1
+ continue
+ }
+ }
+
+ // Only process further if we are inside a Fragment
+ if (curFragName == null) continue
+
+ // ── Pipeline ───────────────────────────────────────────────────────
+ if (content =~ /^Pipeline \d+\(instance_num=\d+\):$/) {
+ if (pipelineIndent == -1) pipelineIndent = spaces
+ if (spaces == pipelineIndent) {
+ flushPipeline()
+ def m = content =~ /^Pipeline (\d+)\(instance_num=(\d+)\):$/
+ if (m.find()) {
+ curPipeName = "Pipeline ${m.group(1)} (instances=${m.group(2)})"
+ } else {
+ curPipeName = content[0..-2]
+ }
+ // Reset operator-level indent markers per pipeline
+ operatorIndent = -1
+ planInfoIndent = -1
+ planItemIndent = -1
+ counterIndent = -1
+ continue
+ }
+ }
+
+ // Only process further if we are inside a Pipeline
+ if (curPipeName == null) continue
+
+ // ── CommonCounters / CustomCounters ────────────────────────────────
+ // These appear inside an operator block and signal end of PlanInfo.
+ if (content == "CommonCounters:" || content == "CustomCounters:") {
+ if (counterIndent == -1) counterIndent = spaces
+ if (spaces == counterIndent) {
+ inPlanInfo = false
+ continue
+ }
+ }
+
+ // ── PlanInfo marker ────────────────────────────────────────────────
+ if (content == "- PlanInfo") {
+ if (planInfoIndent == -1) planInfoIndent = spaces
+ if (spaces == planInfoIndent) {
+ inPlanInfo = true
+ continue
+ }
+ }
+
+ // ── PlanInfo item ──────────────────────────────────────────────────
+ if (inPlanInfo && content.startsWith("- ")) {
+ if (planItemIndent == -1) planItemIndent = spaces
+ if (spaces == planItemIndent) {
+ curPlanInfo << content.substring(2) // strip leading "- "
+ continue
+ }
+ }
+
+ // ── Operator line ──────────────────────────────────────────────────
+ // Operator names start with an uppercase letter, contain only
+ // A-Z, 0-9, _, (, ) characters, and end with ':'.
+ // Skip counter section headers (CommonCounters / CustomCounters already handled above).
+ if (content.endsWith(":") && content =~ /^[A-Z][A-Z0-9_]/) {
+ // Ignore pure counter/info headers that are not operators
+ if (content == "CommonCounters:" || content == "CustomCounters:" ||
+ content == "PlanInfo:") {
+ continue
+ }
+ if (operatorIndent == -1) operatorIndent = spaces
+ if (spaces == operatorIndent) {
+ flushOp()
+ curOpName = content[0..-2] // strip trailing ':'
+ inPlanInfo = false
+ planInfoIndent = -1
+ planItemIndent = -1
+ counterIndent = -1
+ continue
+ }
+ }
+ }
+
+ flushFragment()
+
+ // ── Format output ──────────────────────────────────────────────────────
+ // Similar to explain plan:
+ // Fragment N:
+ // Pipeline M (instances=K):
+ // OPERATOR_NAME (...)
+ // | plan-info-key: value
+ // | ...
+ if (fragments.isEmpty()) {
+ return "(MergedProfile found but no fragments could be parsed)"
+ }
+
+ def sb = new StringBuilder()
+ for (def frag : fragments) {
+ sb.append("${frag.name}:\n")
+ for (def pipe : frag.pipelines) {
+ sb.append(" ${pipe.name}:\n")
+ for (def op : pipe.ops) {
+ sb.append(" ${op.name}\n")
+ for (def pi : op.planInfo) {
+ sb.append(" | ${pi}\n")
+ }
+ }
+ }
+ }
+
+ return sb.toString()
+}
+
+// ---------------------------------------------------------------------------
+// profile_plan_tree_from_id(queryId) → formatted String
+// Fetches profile via HTTP then calls profile_plan_tree.
+// ---------------------------------------------------------------------------
+Suite.metaClass.profile_plan_tree_from_id = { String queryId ->
+ Suite suite = delegate as Suite
+ def dst = 'http://' + suite.context.config.feHttpAddress
+ def conn = new URL("${dst}/api/profile/text?query_id=${queryId}").openConnection()
+ conn.setRequestMethod("GET")
+ def user = suite.context.config.feHttpUser ?: "root"
+ def pass = suite.context.config.feHttpPassword ?: ""
+ def encoding = Base64.getEncoder().encodeToString("${user}:${pass}".getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ conn.setConnectTimeout(5000)
+ conn.setReadTimeout(15000)
+ def profileText = conn.getInputStream().getText()
+ def tree = suite.profile_plan_tree(profileText)
+ return tree.split('\n').findAll { !it.startsWith(' | ') }.join('\n')
+}
+
+// ---------------------------------------------------------------------------
+// profile_plan_tree_from_sql(testSql) → formatted String
+// Executes the SQL with profiling enabled and SQL cache disabled,
+// waits for the profile to be collected, then returns the operator tree.
+// ---------------------------------------------------------------------------
+Suite.metaClass.profile_plan_tree_from_sql = { String testSql ->
+ Suite suite = delegate as Suite
+ suite.sql "set enable_profile=true;"
+ suite.sql "set enable_sql_cache=false;"
+ suite.sql testSql
+ def qid = suite.sql("select last_query_id()")[0][0] as String
+ Thread.sleep(1500)
+ return suite.profile_plan_tree_from_id(qid)
+}
+
+logger.info("Added 'profile_plan_tree', 'profile_plan_tree_from_id' and 'profile_plan_tree_from_sql' to Suite")
diff --git a/regression-test/suites/nereids_p0/local_shuffle/print_known_diff_trees.groovy b/regression-test/suites/nereids_p0/local_shuffle/print_known_diff_trees.groovy
new file mode 100644
index 00000000000..7ce3fd2352e
--- /dev/null
+++ b/regression-test/suites/nereids_p0/local_shuffle/print_known_diff_trees.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("print_known_diff_trees", "p0") {
+ sql "use regression_test_nereids_p0_local_shuffle"
+
+ sql "set enable_pipeline_engine=true;"
+ sql "set enable_local_shuffle_planner=false;"
+ sql "set disable_join_reorder=true;"
+ sql "set disable_colocate_plan=true;"
+ sql "set ignore_storage_data_distribution=false;"
+ sql "set parallel_pipeline_task_num=4;"
+ sql "set auto_broadcast_join_threshold=-1;"
+ sql "set broadcast_row_count_limit=0;"
+
+ def printBETree = { String tag, String testSql ->
+ def tree = profile_plan_tree_from_sql(testSql)
+ logger.info("=== [${tag}] BE operator tree ===\nSQL: ${testSql.trim()}\n${tree}")
+ }
+
+ printBETree("analytic_no_partition",
+ "SELECT k1, sum(v1) OVER() AS s FROM ls_t1 ORDER BY k1, s")
+
+ printBETree("distinct_non_bucket_key",
+ "SELECT DISTINCT k2 FROM ls_t1 ORDER BY k2")
+
+ printBETree("set_intersect_three_way",
+ """SELECT k1 FROM ls_t1
+ INTERSECT
+ SELECT k1 FROM ls_t2
+ INTERSECT
+ SELECT k1 FROM ls_t3
+ ORDER BY k1""")
+
+ printBETree("union_all_followed_by_agg",
+ """SELECT k1, count(*) AS cnt
+ FROM (
+ SELECT k1, v1 AS v FROM ls_t1
+ UNION ALL
+ SELECT k1, v2 AS v FROM ls_t2
+ ) u
+ GROUP BY k1
+ ORDER BY k1""")
+
+ printBETree("table_function",
+ """SELECT k1, e1 FROM ls_t1
+ LATERAL VIEW explode_numbers(v1) tmp AS e1
+ ORDER BY k1, e1 LIMIT 20""")
+
+ printBETree("agg_after_shuffle_join_non_bucket_key",
+ """SELECT a.k2, count(*) AS cnt
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k2
+ ORDER BY a.k2""")
+
+ printBETree("agg_after_broadcast_join",
+ """SELECT a.k1, count(*) AS cnt
+ FROM ls_t1 a JOIN [broadcast] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k1
+ ORDER BY a.k1""")
+
+ printBETree("agg_finalize_serial_pooling_bucket",
+ "SELECT /*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,ignore_storage_data_distribution=true,parallel_pipeline_task_num=4,auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0)*/k1, count(*) AS cnt FROM ls_serial GROUP BY k1 ORDER BY k1",
+ )
+
+}
diff --git a/regression-test/suites/nereids_p0/local_shuffle/test_local_shuffle_fe_be_consistency.groovy b/regression-test/suites/nereids_p0/local_shuffle/test_local_shuffle_fe_be_consistency.groovy
new file mode 100644
index 00000000000..37bfb7df938
--- /dev/null
+++ b/regression-test/suites/nereids_p0/local_shuffle/test_local_shuffle_fe_be_consistency.groovy
@@ -0,0 +1,940 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/**
+ * Verify FE-planned local exchange (enable_local_shuffle=true) matches
+ * BE-native local exchange (enable_local_shuffle=false) for every operator type.
+ *
+ * Profile operator names produced by BE (from get_exchange_type_name()):
+ * LOCAL_EXCHANGE_SINK_OPERATOR(GLOBAL_HASH_SHUFFLE) ← GLOBAL_EXECUTION_HASH_SHUFFLE
+ * LOCAL_EXCHANGE_SINK_OPERATOR(LOCAL_HASH_SHUFFLE) ← LOCAL_EXECUTION_HASH_SHUFFLE
+ * LOCAL_EXCHANGE_SINK_OPERATOR(BUCKET_HASH_SHUFFLE)
+ * LOCAL_EXCHANGE_SINK_OPERATOR(PASSTHROUGH)
+ * LOCAL_EXCHANGE_SINK_OPERATOR(ADAPTIVE_PASSTHROUGH)
+ * LOCAL_EXCHANGE_SINK_OPERATOR(BROADCAST)
+ * LOCAL_EXCHANGE_SINK_OPERATOR(PASS_TO_ONE)
+ * LOCAL_EXCHANGE_OPERATOR(...) ← source side (same type, not counted)
+ *
+ * Methodology:
+ * 1. Run SQL with enable_local_shuffle=false (BE native) → capture profile via HTTP
+ * 2. Run SQL with enable_local_shuffle=true (FE planned) → capture profile via HTTP
+ * 3. Count LOCAL_EXCHANGE_SINK_OPERATOR entries per type in each profile
+ * 4. Log comparison results (MATCH or MISMATCH)
+ * 5. Assert query results are identical (check_sql_equal)
+ *
+ * Known FE/BE design differences (not bugs, logged as INFO):
+ * - NLJ probe: FE always requires ADAPTIVE_PASSTHROUGH; BE requires NOOP for
+ * RIGHT_OUTER/RIGHT_SEMI/RIGHT_ANTI/FULL_OUTER (FE adds extra exchange for those types)
+ */
+suite("test_local_shuffle_fe_be_consistency") {
+
+ // ============================================================
+ // Helper: fetch profile text via HTTP (root, no password)
+ // ============================================================
+ def getProfile = { String queryId ->
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/api/profile/text?query_id=${queryId}").openConnection()
+ conn.setRequestMethod("GET")
+ def user = context.config.feHttpUser ?: "root"
+ def pass = context.config.feHttpPassword ?: ""
+ def encoding = Base64.getEncoder().encodeToString("${user}:${pass}".getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ conn.setConnectTimeout(5000)
+ conn.setReadTimeout(10000)
+ return conn.getInputStream().getText()
+ }
+
+ // ============================================================
+ // Helper: extract LOCAL_EXCHANGE_SINK_OPERATOR types + counts
+ // Returns a sorted map like [PASSTHROUGH:2, GLOBAL_HASH_SHUFFLE:1]
+ // ============================================================
+ def extractSinkExchangeCounts = { String profileText ->
+ def counts = [:]
+ profileText.eachLine { line ->
+ // matches: LOCAL_EXCHANGE_SINK_OPERATOR(PASSTHROUGH) or with leading spaces/dashes
+ def m = line =~ /LOCAL_EXCHANGE_SINK_OPERATOR\(([^)]+)\)/
+ if (m.find()) {
+ def type = m.group(1).trim()
+ counts[type] = (counts[type] ?: 0) + 1
+ }
+ }
+ return counts.sort()
+ }
+
+ // ============================================================
+ // Helper: run SQL, get query_id, fetch profile, extract LE counts
+ // enableFePlanner=true → enable_local_shuffle_planner=true (FE plans exchanges)
+ // enableFePlanner=false → enable_local_shuffle_planner=false (BE plans natively)
+ // ============================================================
+ // Poll until the profile contains "Is Profile Collection Completed: true".
+ // This field is written by SummaryProfile.queryFinished() after waitForFragmentsDone(),
+ // guaranteeing all BE operator metrics have been merged into the profile.
+ def waitForProfile = { String queryId ->
+ // Poll until the profile is fully collected. The "Is Profile Collection
+ // Completed: true" marker is written after all BE fragments have reported,
+ // so once it appears the LOCAL_EXCHANGE_SINK counts are final.
+ def maxAttempts = 60
+ def sleepMs = 200
+ for (int i = 0; i < maxAttempts; i++) {
+ Thread.sleep(sleepMs)
+ try {
+ def text = getProfile(queryId)
+ if (text.contains("Is Profile Collection Completed: true")) {
+ return text
+ }
+ } catch (Exception ignored) {}
+ }
+ return getProfile(queryId)
+ }
+
+ // Execute SQL and return query_id (no profile wait — fast)
+ def runAndGetQueryId = { String testSql, boolean enableFePlanner ->
+ sql "set enable_profile=true"
+ sql "set enable_local_shuffle_planner=${enableFePlanner}"
+ sql "set enable_sql_cache=false"
+ sql "${testSql}"
+ def queryIdResult = sql "select last_query_id()"
+ return queryIdResult[0][0].toString()
+ }
+
+ // ============================================================
+ // Two-phase approach for speed:
+ // Phase 1: Execute all queries (BE + FE) sequentially, collect query IDs
+ // Phase 2: Fetch all profiles in parallel (profiles collected in background)
+ // This overlaps profile collection with query execution, avoiding serial waits.
+ // ============================================================
+ def mismatches = Collections.synchronizedList([])
+ def pendingChecks = Collections.synchronizedList([])
+
+ def setVarBase = "disable_join_reorder=true,disable_colocate_plan=true,ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0"
+
+ def checkConsistencyWithSql = { String tag, String testSql, boolean knownDiff = false ->
+ // Phase 1: execute queries and collect query IDs (fast, no profile wait)
+ String beQueryId = null
+ String feQueryId = null
+ boolean beFailed = false
+ boolean feFailed = false
+ String beError = null
+ String feError = null
+
+ try {
+ beQueryId = runAndGetQueryId(testSql, false)
+ } catch (Throwable t) {
+ beFailed = true
+ beError = t.message
+ }
+ try {
+ feQueryId = runAndGetQueryId(testSql, true)
+ } catch (Throwable t) {
+ feFailed = true
+ feError = t.message
+ }
+
+ // Build SET_VAR versions for check_sql_equal
+ def sqlOn = testSql.replaceFirst(/(?i)\/\*\+SET_VAR\(([^)]*)\)\s*\*\//, "/*+SET_VAR(enable_local_shuffle_planner=true,\$1)*/")
+ def sqlOff = testSql.replaceFirst(/(?i)\/\*\+SET_VAR\(([^)]*)\)\s*\*\//, "/*+SET_VAR(enable_local_shuffle_planner=false,\$1)*/")
+ if (!testSql.contains("/*+SET_VAR")) {
+ sqlOn = testSql.replaceFirst(/(?i)^\s*(SELECT)\s+/, "SELECT /*+SET_VAR(enable_local_shuffle_planner=true,${setVarBase})*/ ")
+ sqlOff = testSql.replaceFirst(/(?i)^\s*(SELECT)\s+/, "SELECT /*+SET_VAR(enable_local_shuffle_planner=false,${setVarBase})*/ ")
+ }
+
+ // Save for phase 2 (deferred profile fetch + comparison)
+ pendingChecks << [tag: tag, beQueryId: beQueryId, feQueryId: feQueryId,
+ beFailed: beFailed, feFailed: feFailed,
+ beError: beError, feError: feError,
+ knownDiff: knownDiff, sqlOn: sqlOn, sqlOff: sqlOff]
+ }
+
+ // Phase 2: fetch profiles and compare (called after all queries are submitted)
+ def resolveAllChecks = {
+ for (def check : pendingChecks) {
+ def tag = check.tag
+ if (check.beFailed) {
+ def errMsg = "[${tag}] BE run FAILED: ${check.beError}"
+ logger.warn(errMsg)
+ if (!check.knownDiff) { mismatches << errMsg }
+ continue
+ }
+ if (check.feFailed) {
+ def errMsg = "[${tag}] FE run FAILED: ${check.feError}"
+ logger.warn(errMsg)
+ if (!check.knownDiff) { mismatches << errMsg }
+ continue
+ }
+
+ // Fetch profiles (likely already collected by now)
+ def beProfile = waitForProfile(check.beQueryId)
+ def feProfile = waitForProfile(check.feQueryId)
+ def beCounts = extractSinkExchangeCounts(beProfile)
+ def feCounts = extractSinkExchangeCounts(feProfile)
+
+ boolean match = (beCounts == feCounts)
+ if (match) {
+ logger.info("[${tag}] MATCH: ${beCounts}")
+ } else {
+ def msg = "[${tag}] ${check.knownDiff ? 'KNOWN-DIFF' : 'MISMATCH'}: BE=${beCounts}, FE=${feCounts}"
+ if (check.knownDiff) {
+ logger.info(msg)
+ } else {
+ logger.warn(msg)
+ mismatches << msg
+ }
+ }
+
+ // Verify result correctness
+ try {
+ check_sql_equal(check.sqlOn, check.sqlOff)
+ } catch (Throwable t) {
+ def errMsg = "[${tag}] check_sql_equal FAILED: ${t.message}"
+ logger.warn(errMsg)
+ mismatches << errMsg
+ }
+ }
+ }
+
+ // ============================================================
+ // Common settings
+ // ============================================================
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "SET runtime_filter_mode=off"
+ sql "SET parallel_pipeline_task_num=4"
+ sql "SET enable_profile=true"
+ sql "SET enable_sql_cache=false"
+ // Keep local shuffle feature globally enabled; only toggle the planner flag
+ sql "SET enable_local_shuffle=true"
+ // Disable ignore_storage_data_distribution to get predictable plans from scans
+ sql "SET ignore_storage_data_distribution=false"
+
+ // ============================================================
+ // Table setup
+ // ls_t1: HASH(k1) 8 buckets
+ // ls_t2: HASH(k1) 8 buckets (same distribution → colocate-eligible)
+ // ls_t3: HASH(k4) 5 buckets (different distribution)
+ // ls_serial: HASH(k1) 2 buckets (for serial-scan tests: 2 < parallel_pipeline_task_num=4)
+ // ============================================================
+ sql "DROP TABLE IF EXISTS ls_t1"
+ sql "DROP TABLE IF EXISTS ls_t2"
+ sql "DROP TABLE IF EXISTS ls_t3"
+ sql "DROP TABLE IF EXISTS ls_serial"
+
+ sql """
+ CREATE TABLE ls_t1 (
+ k1 INT NOT NULL,
+ k2 INT,
+ v1 INT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k1, k2)
+ DISTRIBUTED BY HASH(k1) BUCKETS 8
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE ls_t2 (
+ k1 INT NOT NULL,
+ k3 INT,
+ v2 INT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k1, k3)
+ DISTRIBUTED BY HASH(k1) BUCKETS 8
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE ls_t3 (
+ k1 INT NOT NULL,
+ k4 INT,
+ v3 INT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k1, k4)
+ DISTRIBUTED BY HASH(k4) BUCKETS 5
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO ls_t1 VALUES
+ (1, 10, 2), (1, 11, 3), (2, 20, 4), (2, 21, 1),
+ (3, 30, 5), (4, 40, 6), (5, 50, 7), (6, 60, 8),
+ (7, 70, 9), (8, 80, 10), (9, 90, 11), (10, 100, 12)
+ """
+
+ sql """
+ INSERT INTO ls_t2 VALUES
+ (1, 100, 7), (1, 101, 1), (2, 200, 2), (3, 300, 3),
+ (4, 400, 4), (5, 500, 5), (6, 600, 6), (7, 700, 7)
+ """
+
+ sql """
+ INSERT INTO ls_t3 VALUES
+ (1, 1001, 5), (1, 1001, 6), (2, 1002, 7),
+ (3, 1003, 8), (4, 1004, 9), (5, 1005, 10)
+ """
+
+ sql """
+ CREATE TABLE ls_serial (
+ k1 INT NOT NULL,
+ k2 INT,
+ v1 INT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k1, k2)
+ DISTRIBUTED BY HASH(k1) BUCKETS 2
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO ls_serial VALUES
+ (1, 10, 2), (2, 20, 4), (3, 30, 5), (4, 40, 6)
+ """
+
+ // SET_VAR prefix used in most test SQLs (disables plan reorder/colocate for deterministic plans)
+ def sv = "/*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0)*/"
+ // Same as sv but forces serial source path (default in many environments)
+ def svSerialSource = "/*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,ignore_storage_data_distribution=true,parallel_pipeline_task_num=4,auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0)*/"
+
+ // ================================================================
+ // Section 1: AggSink / StreamingAgg scenarios
+ // BE operator: AggSinkOperatorX / StreamingAggOperatorX
+ // ================================================================
+
+ // 1-1: AggSink finalize, no group key
+ // Finalize phase: FE uses noRequire() (needsFinalize && groupingExprs.isEmpty()) → NOOP
+ // Streaming pre-agg phase: FE uses requirePassthrough() (else branch, groupingExprs.isEmpty())
+ // BE: does not add exchanges for no-group-key agg (NOOP)
+ // Known diff: FE inserts PASSTHROUGH for streaming pre-agg; BE skips exchanges entirely
+ checkConsistencyWithSql("agg_finalize_no_group_key",
+ "SELECT ${sv} count(*) FROM ls_t1")
+
+ // 1-2: AggSink 1-phase, bucket key (k1) → no mismatch (distribution matches)
+ // BE: BUCKET_HASH_SHUFFLE for sink, scan already provides BUCKET_HASH_SHUFFLE
+ // → need_to_local_exchange returns false (both hash types match)
+ checkConsistencyWithSql("agg_1phase_bucket_key",
+ "SELECT ${sv} k1, count(*) AS cnt FROM ls_t1 GROUP BY k1 ORDER BY k1")
+
+ // 1-2b: Same SQL under serial-source mode (ignore_storage_data_distribution=true)
+ // This explicitly validates FE/BE consistency under serial-source planning path.
+ checkConsistencyWithSql("agg_1phase_bucket_key_serial_source",
+ "SELECT ${svSerialSource} k1, count(*) AS cnt FROM ls_t1 GROUP BY k1 ORDER BY k1")
+
+ // 1-2c: Finalize agg, serial/pooling scan, bucket key (k1), ls_serial (2 buckets).
+ // Pooling scan + bucket-key colocate agg: BE inserts PASSTHROUGH fan-out (heavy_ops
+ // bottleneck avoidance before LOCAL_HASH_SHUFFLE) + LOCAL_HASH_SHUFFLE.
+ // FE mirrors with heavy_ops check in enforceChild.
+ checkConsistencyWithSql("agg_finalize_serial_pooling_bucket",
+ "SELECT ${svSerialSource} k1, count(*) AS cnt FROM ls_serial GROUP BY k1 ORDER BY k1")
+
+ // 1-2c2: Same finalize agg with bucket key, but non-pooling (ignore_storage_data_distribution=false).
+ // No serial source → no heavy_ops PASSTHROUGH fan-out needed.
+ // Known mismatch on clusters where ls_serial (2 BUCKETS) tablets land on different BEs:
+ // FE sees global instanceCount=2 → inserts LE; each BE sees local _num_instances=1 → skips.
+ // This is a pre-existing FE/BE instanceCount discrepancy, not a planner bug.
+ checkConsistencyWithSql("agg_finalize_non_pooling_bucket",
+ "SELECT ${sv} k1, count(*) AS cnt FROM ls_serial GROUP BY k1 ORDER BY k1")
+
+ // 1-2d: Agg, serial/pooling scan, non-bucket key (k2), ls_serial.
+ checkConsistencyWithSql("agg_finalize_serial_pooling_non_bucket",
+ "SELECT ${svSerialSource} k2, count(*) AS cnt FROM ls_serial GROUP BY k2 ORDER BY k2")
+
+ // 1-3: AggSink 1-phase, non-bucket key (k2)
+ // BE: GLOBAL_EXECUTION_HASH_SHUFFLE vs BUCKET_HASH_SHUFFLE from scan
+ // → inserts GLOBAL_HASH_SHUFFLE (or LOCAL_HASH_SHUFFLE for local execution)
+ checkConsistencyWithSql("agg_1phase_non_bucket_key",
+ "SELECT ${sv} k2, count(*) AS cnt FROM ls_t1 GROUP BY k2 ORDER BY k2")
+
+ // 1-4: AggSink 1-phase, multi-column non-bucket key (k1,k2)
+ // Even though k1 is the bucket key, (k1,k2) is not → GLOBAL_HASH_SHUFFLE
+ checkConsistencyWithSql("agg_1phase_multi_key_non_bucket",
+ "SELECT ${sv} k1, k2, count(*) AS cnt FROM ls_t1 GROUP BY k1, k2 ORDER BY k1, k2")
+
+ // 1-5: Two-phase agg (pre-agg + finalize), non-bucket key
+ // Tests that both streaming agg pre-phase and finalize phases are handled correctly
+ checkConsistencyWithSql("agg_two_phase_non_bucket",
+ "SELECT ${sv} k2, sum(v1) AS s FROM ls_t1 GROUP BY k2 ORDER BY k2")
+
+ // ================================================================
+ // Section 2: DistinctStreamingAgg scenarios
+ // BE operator: DistinctStreamingAggOperatorX
+ // ================================================================
+
+ // 2-1: DISTINCT on bucket key → no extra exchange needed (distribution matches)
+ checkConsistencyWithSql("distinct_bucket_key",
+ "SELECT ${sv} DISTINCT k1 FROM ls_t1 ORDER BY k1")
+
+ // 2-2: DISTINCT on non-bucket key
+ checkConsistencyWithSql("distinct_non_bucket_key",
+ "SELECT ${sv} DISTINCT k2 FROM ls_t1 ORDER BY k2")
+
+ // 2-3: DISTINCT on multiple non-bucket keys
+ checkConsistencyWithSql("distinct_multi_non_bucket",
+ "SELECT ${sv} DISTINCT k1, k2 FROM ls_t1 ORDER BY k1, k2")
+
+ // ================================================================
+ // Section 3: AnalyticSink / SortSink (analytic) scenarios
+ // BE operators: AnalyticSinkOperatorX, SortSinkOperatorX
+ // ================================================================
+
+ // 3-1: Analytic window, no PARTITION BY → serial path, no exchange needed
+ checkConsistencyWithSql("analytic_no_partition",
+ "SELECT ${sv} k1, sum(v1) OVER() AS s FROM ls_t1 ORDER BY k1, s")
+
+ // 3-2: Analytic window, PARTITION BY non-bucket key → GLOBAL_HASH_SHUFFLE
+ // Also triggers SortSink (analytic sort) → GLOBAL_HASH_SHUFFLE
+ checkConsistencyWithSql("analytic_partition_non_bucket",
+ "SELECT ${sv} k1, k2, row_number() OVER(PARTITION BY k2 ORDER BY k1) AS rn FROM ls_t1 ORDER BY k2, k1, rn")
+
+ // 3-3: Analytic window, PARTITION BY bucket key → BUCKET_HASH_SHUFFLE (or no extra exchange)
+ // SortSink(analytic): if colocate+bucket → BUCKET_HASH_SHUFFLE
+ checkConsistencyWithSql("analytic_partition_bucket_key",
+ "SELECT ${sv} k1, sum(v1) OVER(PARTITION BY k1) AS s FROM ls_t1 ORDER BY k1, s")
+
+ // 3-4: ORDER BY sort (SortSink._merge_by_exchange=true) → PASSTHROUGH
+ checkConsistencyWithSql("sort_order_by",
+ "SELECT ${sv} * FROM ls_t1 ORDER BY k1, k2 LIMIT 10")
+
+ // ================================================================
+ // Section 4: PartitionSortSink scenarios
+ // BE operator: PartitionSortSinkOperatorX
+ // ================================================================
+
+ // 4-1: PartitionSort TWO_PHASE_GLOBAL (triggered by QUALIFY / ROW_NUMBER with LIMIT)
+ // → GLOBAL_EXECUTION_HASH_SHUFFLE
+ checkConsistencyWithSql("partition_sort_two_phase_global",
+ """SELECT ${sv} k1, k2, v1
+ FROM (
+ SELECT k1, k2, v1,
+ ROW_NUMBER() OVER(PARTITION BY k2 ORDER BY v1 DESC) AS rn
+ FROM ls_t1
+ ) t
+ WHERE rn <= 2
+ ORDER BY k1, k2""")
+
+ // 4-2: PartitionSort single phase (TWO_PHASE_LOCAL or ONE_PHASE) → PASSTHROUGH
+ // Note: This depends on the optimizer's choice; TopN on non-partitioned window
+ checkConsistencyWithSql("partition_sort_single_phase",
+ """SELECT ${sv} k1, k2, v1
+ FROM (
+ SELECT k1, k2, v1,
+ ROW_NUMBER() OVER(PARTITION BY k2 ORDER BY v1 DESC) AS rn
+ FROM ls_t1
+ ) t
+ WHERE rn = 1
+ ORDER BY k1, k2""")
+
+ // ================================================================
+ // Section 5: HashJoinProbe / HashJoinBuildSink scenarios
+ // BE operators: HashJoinProbeOperatorX, HashJoinBuildSinkOperatorX
+ // ================================================================
+
+ // 5-1: Broadcast join — probe NOOP (or PASSTHROUGH if serial), build PASS_TO_ONE (if serial)
+ checkConsistencyWithSql("hash_join_broadcast",
+ """SELECT ${sv} a.k1, a.v1, b.v2
+ FROM ls_t1 a JOIN [broadcast] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1, a.k2""")
+
+ // 5-2: Shuffle (PARTITIONED) join → probe GLOBAL_HASH_SHUFFLE, build GLOBAL_HASH_SHUFFLE
+ checkConsistencyWithSql("hash_join_shuffle",
+ """SELECT ${sv} a.k1, a.v1, b.v2
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1, a.k2""")
+
+ // 5-3: LEFT OUTER shuffle join
+ // Known diff: In single-BE environments, BE's pipeline-level need_to_local_exchange()
+ // may skip exchanges when num_tasks_of_parent<=1, while FE still inserts PASSTHROUGH
+ // because it lacks pipeline-level task count information.
+ checkConsistencyWithSql("hash_join_left_outer_shuffle",
+ """SELECT ${sv} a.k1, a.v1, b.v2
+ FROM ls_t1 a LEFT OUTER JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1, a.k2""")
+
+ // 5-4: RIGHT OUTER shuffle join
+ checkConsistencyWithSql("hash_join_right_outer_shuffle",
+ """SELECT ${sv} a.k1, a.v1, b.v2
+ FROM ls_t1 a RIGHT OUTER JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1, b.k3""")
+
+ // 5-5: FULL OUTER shuffle join
+ checkConsistencyWithSql("hash_join_full_outer_shuffle",
+ """SELECT ${sv} a.k1, a.v1, b.v2
+ FROM ls_t1 a FULL OUTER JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1, b.k3""")
+
+ // 5-6: LEFT SEMI shuffle join
+ checkConsistencyWithSql("hash_join_left_semi_shuffle",
+ """SELECT ${sv} a.k1, a.v1
+ FROM ls_t1 a LEFT SEMI JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1""")
+
+ // 5-7: LEFT ANTI shuffle join
+ // Known diff: Same as LEFT OUTER — BE pipeline-level num_tasks_of_parent<=1 check
+ // skips exchanges in single-BE environments; FE cannot replicate this.
+ checkConsistencyWithSql("hash_join_left_anti_shuffle",
+ """SELECT ${sv} a.k1, a.v1
+ FROM ls_t1 a LEFT ANTI JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1""")
+
+ // 5-8: NULL_AWARE_LEFT_ANTI_JOIN (NOT IN subquery with nullable) → NOOP
+ // BE: both probe and build NOOP
+ checkConsistencyWithSql("hash_join_null_aware_left_anti",
+ """SELECT ${sv} k1, v1 FROM ls_t1
+ WHERE k1 NOT IN (SELECT k1 FROM ls_t2)
+ ORDER BY k1""")
+
+ // ================================================================
+ // Section 6: NestedLoopJoin scenarios
+ // BE operators: NestedLoopJoinProbeOperatorX, NestedLoopJoinBuildSinkOperatorX
+ // ================================================================
+
+ // 6-1: NLJ INNER (cross/theta join) → probe ADAPTIVE_PASSTHROUGH, build BROADCAST (serial)
+ // FE: requireAdaptivePassthrough for probe; BE: ADAPTIVE_PASSTHROUGH for probe
+ checkConsistencyWithSql("nlj_inner_theta",
+ """SELECT ${sv} a.k1, b.k1 AS bk1
+ FROM ls_t1 a, ls_t2 b WHERE a.k1 > b.k1
+ ORDER BY a.k1, bk1 LIMIT 20""")
+
+ // 6-2: NLJ LEFT OUTER
+ checkConsistencyWithSql("nlj_left_outer",
+ """SELECT ${sv} a.k1, b.k1 AS bk1
+ FROM ls_t1 a LEFT OUTER JOIN ls_t2 b ON a.k1 > b.k1
+ ORDER BY a.k1, bk1 LIMIT 20""")
+
+ // 6-3: NLJ RIGHT OUTER → BE probe: NOOP; FE: ADAPTIVE_PASSTHROUGH (known difference)
+ // FE uses requireAdaptivePassthrough unconditionally for non-NULL_AWARE NLJ
+ // BE uses NOOP for RIGHT_OUTER/RIGHT_SEMI/RIGHT_ANTI/FULL_OUTER
+ checkConsistencyWithSql("nlj_right_outer", """
+ SELECT ${sv} a.k1, b.k1 AS bk1
+ FROM ls_t1 a RIGHT OUTER JOIN ls_t2 b ON a.k1 < b.k1
+ ORDER BY a.k1, bk1 LIMIT 20
+ """)
+
+ // 6-4: NLJ FULL OUTER → same known difference as RIGHT_OUTER
+ checkConsistencyWithSql("nlj_full_outer", """
+ SELECT ${sv} a.k1, b.k1 AS bk1
+ FROM ls_t1 a FULL OUTER JOIN ls_t2 b ON a.k1 > b.k1
+ ORDER BY a.k1, bk1 LIMIT 20
+ """)
+
+ // ================================================================
+ // Section 7: Set operations (INTERSECT / EXCEPT)
+ // BE operators: SetSinkOperatorX, SetProbeSinkOperatorX, SetSourceOperatorX
+ // ================================================================
+
+ // 7-1: INTERSECT → GLOBAL_HASH_SHUFFLE for both set sink and set probe sink
+ checkConsistencyWithSql("set_intersect",
+ """SELECT ${sv} k1 FROM ls_t1
+ INTERSECT
+ SELECT k1 FROM ls_t2
+ ORDER BY k1""")
+
+ // 7-2: EXCEPT → GLOBAL_HASH_SHUFFLE
+ checkConsistencyWithSql("set_except",
+ """SELECT ${sv} k1 FROM ls_t1
+ EXCEPT
+ SELECT k1 FROM ls_t2
+ ORDER BY k1""")
+
+ // 7-3: Three-way INTERSECT
+ // FE and BE are consistent:
+ // - ls_t1/ls_t2 (colocated on k1): DISTINCT_STREAMING_AGG with needsFinalize=true
+ // → BE returns NOOP/HASH (already satisfied), FE requireHash (already satisfied) → no exchange
+ // - ls_t3 (non-colocated, partition on k4): DISTINCT_STREAMING_AGG with needsFinalize=false
+ // → BE returns PASSTHROUGH (enable_distinct_streaming_agg_force_passthrough=true),
+ // FE enableDistinctStreamingAggForcePassthrough=true → requirePassthrough → insert PASSTHROUGH
+ checkConsistencyWithSql("set_intersect_three_way",
+ """SELECT ${sv} k1 FROM ls_t1
+ INTERSECT
+ SELECT k1 FROM ls_t2
+ INTERSECT
+ SELECT k1 FROM ls_t3
+ ORDER BY k1""")
+
+ // ================================================================
+ // Section 8: UNION scenarios
+ // BE operators: UnionSinkOperatorX, UnionSourceOperatorX
+ // ================================================================
+
+ // 8-1: UNION ALL (no downstream shuffled op → base default)
+ checkConsistencyWithSql("union_all_simple",
+ """SELECT ${sv} k1, v1 FROM ls_t1
+ UNION ALL
+ SELECT k1, v2 FROM ls_t2
+ ORDER BY k1, v1""")
+
+ // 8-2: UNION ALL feeding into GROUP BY (union followed by shuffled agg)
+ checkConsistencyWithSql("union_all_followed_by_agg",
+ """SELECT ${sv} k1, count(*) AS cnt
+ FROM (
+ SELECT k1, v1 AS v FROM ls_t1
+ UNION ALL
+ SELECT k1, v2 AS v FROM ls_t2
+ ) u
+ GROUP BY k1
+ ORDER BY k1""")
+
+ // 8-3: UNION followed by analytic window
+ checkConsistencyWithSql("union_all_followed_by_window",
+ """SELECT ${sv} k1, SUM(v) OVER(PARTITION BY k1) AS sv
+ FROM (
+ SELECT k1, v1 AS v FROM ls_t1
+ UNION ALL
+ SELECT k1, v2 AS v FROM ls_t2
+ ) u
+ ORDER BY k1, sv""")
+
+ // ================================================================
+ // Section 9: TableFunction and AssertNumRows
+ // BE operators: TableFunctionOperatorX, AssertNumRowsOperatorX
+ // ================================================================
+
+ // 9-1: TableFunction (non-pooling) → PASSTHROUGH×2
+ // BE TableFunctionOperatorX overrides required_data_distribution() to always return
+ // PASSTHROUGH; need_to_local_exchange Step 4 always inserts non-hash exchanges.
+ // So: OlapScan → PT → TableFunc → PT → Sort. Total: 2 PASSTHROUGH.
+ // FE mirrors: TableFunctionNode requires PASSTHROUGH from child (outputs NOOP),
+ // SortNode independently inserts PASSTHROUGH for mergeByExchange.
+ checkConsistencyWithSql("table_function",
+ """SELECT ${sv} k1, e1 FROM ls_t1
+ LATERAL VIEW explode_numbers(v1) tmp AS e1
+ ORDER BY k1, e1 LIMIT 20""")
+
+ // 9-1b: TableFunction (pooling scan) → PASSTHROUGH×2
+ // Same as 9-1: TableFunctionOperatorX always requires PASSTHROUGH regardless of child.
+ // Pooling scan (serial) → PT fan-out → TableFunc → PT → Sort. Total: 2 PASSTHROUGH.
+ // FE mirrors: TableFunctionNode requires PASSTHROUGH (outputs NOOP),
+ // SortNode independently inserts PASSTHROUGH for mergeByExchange.
+ checkConsistencyWithSql("table_function_pooling",
+ """SELECT ${svSerialSource} k1, e1 FROM ls_t1
+ LATERAL VIEW explode_numbers(v1) tmp AS e1
+ ORDER BY k1, e1 LIMIT 20""")
+
+ // 9-2: AssertNumRows (scalar subquery) → PASSTHROUGH
+ // Known diff: In single-BE environments, FE and BE may disagree on instance counts
+ // for fragments containing AssertNumRows, leading to different exchange decisions.
+ checkConsistencyWithSql("assert_num_rows",
+ """SELECT ${sv} k1, (SELECT count(*) FROM ls_t2 WHERE ls_t2.k1 = ls_t1.k1) AS cnt
+ FROM ls_t1
+ ORDER BY k1""")
+
+ // ================================================================
+ // Section 10: Mixed / multi-level scenarios
+ // ================================================================
+
+ // 10-1: Agg after shuffle join (k1 is bucket key → no extra exchange after join)
+ checkConsistencyWithSql("agg_after_shuffle_join_bucket_key",
+ """SELECT ${sv} a.k1, count(*) AS cnt
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k1
+ ORDER BY a.k1""")
+
+ // 10-2: Agg after shuffle join on non-bucket column
+ // GROUP BY k2 ≠ join key k1. BE's StreamingAggOperatorX sees child is_hash_join_probe()
+ // and returns PASSTHROUGH (enable_streaming_agg_hash_join_force_passthrough=true by default),
+ // splitting the pipeline at the streaming pre-agg/join boundary.
+ // FE replicates this: AggregationNode detects useStreamingPreagg && child is HashJoinNode
+ // → requirePassthrough. Both FE and BE produce 18 PASSTHROUGH exchanges.
+ checkConsistencyWithSql("agg_after_shuffle_join_non_bucket_key",
+ """SELECT ${sv} a.k2, count(*) AS cnt
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k2
+ ORDER BY a.k2""")
+
+ // 10-3: Agg after broadcast join
+ // ScanNode returns BUCKET_HASH_SHUFFLE (mirroring BE's ScanOperator). HashJoinNode
+ // (broadcast, non-serial probe) propagates probe side's distribution as its own
+ // output type (instead of hardcoding NOOP). AggNode sees BUCKET_HASH_SHUFFLE,
+ // RequireHash.satisfy(BUCKET_HASH_SHUFFLE)=true → no redundant hash exchange.
+ // Mirrors BE's !(hash && hash) check in need_to_local_exchange.
+ checkConsistencyWithSql("agg_after_broadcast_join",
+ """SELECT ${sv} a.k1, count(*) AS cnt
+ FROM ls_t1 a JOIN [broadcast] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k1
+ ORDER BY a.k1""")
+
+ // 10-4: Window after UNION
+ // Known diff: In single-BE environments, FE instance-count-based skipping and
+ // BE pipeline-level num_tasks checks can diverge for union+window fragments.
+ checkConsistencyWithSql("window_after_union",
+ """SELECT ${sv} k1, SUM(v) OVER(PARTITION BY k1) AS sv
+ FROM (
+ SELECT k1, v1 AS v FROM ls_t1
+ UNION ALL
+ SELECT k1, v2 AS v FROM ls_t2
+ ) u
+ ORDER BY k1, sv""")
+
+ // 10-5: Multi-level join + agg + window
+ checkConsistencyWithSql("join_agg_window_multilevel",
+ """SELECT ${sv} t.k1, t.cnt,
+ row_number() OVER(ORDER BY t.cnt DESC, t.k1 ASC) AS rn
+ FROM (
+ SELECT a.k1, count(*) AS cnt
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k1
+ ) t
+ ORDER BY t.k1""")
+
+ // 10-6: Two shuffle joins chained
+ checkConsistencyWithSql("two_shuffle_joins",
+ """SELECT ${sv} a.k1, b.k3, c.k4
+ FROM ls_t1 a
+ JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ JOIN [shuffle] ls_t3 c ON a.k1 = c.k1
+ ORDER BY a.k1, b.k3 LIMIT 20""")
+
+ // 10-7: Complex: join → agg → agg (double-layer aggregation after join)
+ // Known diff: Multi-level agg fragments may have single instances in single-BE
+ // environments, causing FE/BE exchange decision divergence.
+ checkConsistencyWithSql("complex_join_double_agg",
+ """SELECT ${sv} z.k1, SUM(z.metric) AS total
+ FROM (
+ SELECT y.k1, SUM(y.metric) AS metric
+ FROM (
+ SELECT a.k1, SUM(a.v1 + b.v2) AS metric
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ GROUP BY a.k1
+ ) y
+ GROUP BY y.k1
+ ) z
+ GROUP BY z.k1
+ ORDER BY z.k1""")
+
+ // 10-8: Agg then INTERSECT
+ checkConsistencyWithSql("agg_then_intersect",
+ """SELECT ${sv} k1
+ FROM (SELECT k1, count(*) AS cnt FROM ls_t1 GROUP BY k1 HAVING cnt > 0) a
+ INTERSECT
+ SELECT k1
+ FROM (SELECT k1, count(*) AS cnt FROM ls_t2 GROUP BY k1 HAVING cnt > 0) b
+ ORDER BY k1""")
+
+ // 10-9: Shuffle join then DISTINCT
+ checkConsistencyWithSql("shuffle_join_then_distinct",
+ """SELECT ${sv} DISTINCT a.k1
+ FROM ls_t1 a JOIN [shuffle] ls_t2 b ON a.k1 = b.k1
+ ORDER BY a.k1""")
+
+ // ================================================================
+ // Section 11: AggSink LOCAL_HASH_SHUFFLE scenarios
+ // Scenarios where BE's need_to_local_exchange() inserts GLOBAL/BUCKET
+ // hash exchange because the source distribution is not hash-compatible.
+ //
+ // Key rule in pipeline.cpp need_to_local_exchange():
+ // If source is BUCKET_HASH and sink requires GLOBAL_HASH → both are hash
+ // → need_to_local_exchange returns false → NO local exchange.
+ // But if source is PASSTHROUGH/NOOP → not both hash → insert GLOBAL_HASH.
+ // ================================================================
+
+ // 11-1: force_to_local_shuffle=true + non-bucket finalize agg
+ // With force_to_local_shuffle, OlapScanNode.isSerialOperator()=true even with 8 tablets.
+ // Optimizer puts agg in a separate finalize fragment receiving hash-partitioned data,
+ // so no LOCAL_HASH_SHUFFLE is generated — only PASSTHROUGH for the NLJ/scan boundary.
+ checkConsistencyWithSql("agg_finalize_force_local_shuffle_non_bucket",
+ """SELECT /*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,
+ ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,
+ force_to_local_shuffle=true,enable_local_shuffle=true)*/ k2, count(*) AS cnt
+ FROM ls_t1 GROUP BY k2 ORDER BY k2""")
+
+ // 11-2: force_to_local_shuffle=true + bucket-key finalize agg
+ // GROUP BY k1 (bucket key of ls_t1): colocate agg stays in same fragment as scan.
+ // FE: AggNode is colocate → requireHash. BE: AggSink returns BUCKET_HASH.
+ // Result MATCH: [PASSTHROUGH:9], consistent with other bucket-key colocate cases.
+ checkConsistencyWithSql("agg_finalize_force_local_shuffle_bucket_key",
+ """SELECT /*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,
+ ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,
+ force_to_local_shuffle=true,enable_local_shuffle=true)*/ k1, count(*) AS cnt
+ FROM ls_t1 GROUP BY k1 ORDER BY k1""")
+
+ // 11-3: NLJ (theta join) → finalize agg on non-bucket key
+ // GROUP BY k2 (non-bucket): optimizer puts agg in a separate finalize fragment
+ // receiving data via a hash-partitioned inter-fragment exchange on k2.
+ // Within each fragment, the distributions are compatible → no LOCAL_HASH_SHUFFLE.
+ // Result MATCH: [ADAPTIVE_PASSTHROUGH:5, PASSTHROUGH:5]
+ checkConsistencyWithSql("agg_after_nlj_non_bucket",
+ """SELECT /*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,
+ ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,
+ auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0)*/ a.k2, count(*) AS cnt
+ FROM ls_t1 a, ls_t2 b WHERE a.k1 > b.k1
+ GROUP BY a.k2 ORDER BY a.k2""")
+
+ // 11-4: NLJ (theta join) → finalize agg on bucket key (LOCAL_HASH_SHUFFLE test)
+ // GROUP BY k1 (bucket key): colocate agg stays in same pipeline as NLJ probe.
+ // The NLJ probe requires ADAPTIVE_PASSTHROUGH → local exchange inserted.
+ // After that exchange, the next pipeline has LocalExchangeSource (PASSTHROUGH distribution)
+ // feeding into AggSink (BUCKET_HASH_SHUFFLE for colocate k1 agg).
+ // PASSTHROUGH source ≠ BUCKET_HASH target, not both-hash → need_to_local_exchange=true
+ // → BE inserts LOCAL_HASH_SHUFFLE (BUCKET type).
+ // FE: AggNode isColocated=true → requireHash → inserts LocalExchangeNode.
+ // Result MATCH: [ADAPTIVE_PASSTHROUGH:9, LOCAL_HASH_SHUFFLE:9, PASSTHROUGH:9]
+ // This is a primary test for regular AggSink generating LOCAL_HASH_SHUFFLE.
+ checkConsistencyWithSql("agg_after_nlj_bucket_key",
+ """SELECT /*+SET_VAR(disable_join_reorder=true,disable_colocate_plan=true,
+ ignore_storage_data_distribution=false,parallel_pipeline_task_num=4,
+ auto_broadcast_join_threshold=-1,broadcast_row_count_limit=0)*/ a.k1, count(*) AS cnt
+ FROM ls_t1 a, ls_t2 b WHERE a.k1 > b.k1
+ GROUP BY a.k1 ORDER BY a.k1""")
+
+ // ================================================================
+ // Section 12: Nested NLJ with pooling scan
+ // Tests that FE correctly inserts local exchange on ALL NLJ build sides,
+ // even when the NLJ's direct children are not ScanNodes (e.g., nested NLJ
+ // or ExchangeNode). Without the fix, the serial Exchange on NLJ(outer)'s
+ // build side would reduce num_tasks to 1, causing "must set shared state,
+ // in CROSS_JOIN_OPERATOR" for instances 1+.
+ // ================================================================
+
+ // 12-1: Nested NLJ with pooling scan — the regression case from RQG.
+ // Two LEFT JOINs with non-equi conditions → two nested NLJ operators.
+ // The outer NLJ's build side is an Exchange (UNPARTITIONED, serial).
+ // FE must insert a BROADCAST local exchange there to fan out to all instances.
+ // Without the fix in NestedLoopJoinNode (removing instanceof ScanNode check),
+ // FE wouldn't insert the local exchange → "must set shared state" error.
+ // BE-native also fails on this query with "_num_remaining_senders: -N",
+ // so we only verify FE mode produces correct results (skip profile comparison).
+ // knownDiff=true to tolerate the BE failure in profile comparison.
+ checkConsistencyWithSql("nested_nlj_pooling_scan",
+ """SELECT ${svSerialSource} count(a.k1) AS cnt, a.v1
+ FROM ls_serial a
+ LEFT JOIN ls_serial b ON b.k2 >= b.k2
+ LEFT JOIN ls_serial c ON b.k1 >= b.k1
+ WHERE a.k1 IS NOT NULL
+ GROUP BY a.v1
+ ORDER BY cnt, a.v1""", true)
+
+ // 12-2: Same nested NLJ but non-pooling (ignore_storage_data_distribution=false).
+ // FE uses manual force-enforce to always insert ADAPTIVE_PASSTHROUGH
+ // on NLJ probe side, matching BE's need_to_local_exchange Step 4 behavior.
+ // Known mismatch on clusters where ls_serial (2 BUCKETS) tablets span multiple BEs:
+ // same FE/BE instanceCount discrepancy as agg_finalize_non_pooling_bucket.
+ checkConsistencyWithSql("nested_nlj_non_pooling",
+ """SELECT ${sv} count(a.k1) AS cnt, a.v1
+ FROM ls_serial a
+ LEFT JOIN ls_serial b ON b.k2 >= b.k2
+ LEFT JOIN ls_serial c ON b.k1 >= b.k1
+ WHERE a.k1 IS NOT NULL
+ GROUP BY a.v1
+ ORDER BY cnt, a.v1""")
+
+ // ================================================================
+ // Section 13: Pooling scan + operators requiring shared state
+ // Regression cases from RQG build 183677 — serial Exchange on build
+ // side of various operators (Agg, Sort, Union/Repeat) reduced pipeline
+ // num_tasks, causing "must set shared state" errors.
+ // Fixed by restoring the num_tasks raise in _create_deferred_local_exchangers
+ // for non-scan serial operators.
+ // ================================================================
+
+ // 13-1: NLJ + AGG with pooling scan.
+ // NLJ creates pipeline boundary; serial Exchange on build side
+ // needs raise to _num_instances for AGG shared state injection.
+ // knownDiff=true: pooling scan + NLJ has FE/BE exchange count
+ // differences (same root cause as nested_nlj_pooling_scan).
+ checkConsistencyWithSql("agg_after_nlj_pooling_scan",
+ """SELECT ${svSerialSource} a.v1, MAX(a.k1) AS mx
+ FROM ls_serial a LEFT JOIN ls_serial b ON b.k2 < b.k2
+ WHERE a.k1 IS NOT NULL
+ GROUP BY a.v1
+ ORDER BY a.v1, mx""", true)
+
+ // 13-2: GROUPING SETS with pooling scan — generates REPEAT (union-like)
+ // operator internally. Serial Exchange reduces num_tasks, causing
+ // "must set shared state, in UNION_OPERATOR / SORT_OPERATOR".
+ // Known issue: deadlocks on clusters where ls_serial (2 BUCKETS) tablets span
+ // multiple BEs — FE inserts LE (global instanceCount=2) but each BE has
+ // _num_instances=1 causing pipeline task mismatch. Pre-existing FE/BE discrepancy.
+ checkConsistencyWithSql("grouping_sets_pooling_scan",
+ """SELECT ${svSerialSource} k1, k2, SUM(v1) AS sv
+ FROM ls_serial
+ GROUP BY GROUPING SETS ((k1, k2), (k1), ())
+ ORDER BY k1, k2, sv""")
+
+ // 13-3: Window function + GROUPING SETS with pooling scan.
+ // Combines analytic (Sort shared state) and GROUPING SETS (Repeat/Union)
+ // — both need correct num_tasks for shared state injection.
+ // Same instanceCount discrepancy as grouping_sets_pooling_scan.
+ checkConsistencyWithSql("window_grouping_sets_pooling_scan",
+ """SELECT ${svSerialSource} k1, SUM(v1),
+ ROW_NUMBER() OVER (ORDER BY k1) AS rn
+ FROM ls_serial
+ GROUP BY GROUPING SETS ((k1), ())
+ ORDER BY k1, rn""")
+
+ // ============================================================
+ // 14. RQG bug cases — serial NLJ + pooling scan (Bug 13 from rqg_bugs)
+ // Serial NLJ (RIGHT_OUTER) with pooling scan. Previously crashed because
+ // FE inserted BROADCAST on build side inflating num_tasks while probe stayed
+ // serial. Fixed: serial NLJ sets buildSideRequire=noRequire().
+ // ============================================================
+ checkConsistencyWithSql("rqg_serial_nlj_right_outer_pooling",
+ """SELECT /*+SET_VAR(use_serial_exchange=true, parallel_pipeline_task_num=0,
+ ignore_storage_data_distribution=true,
+ enable_share_hash_table_for_broadcast_join=false,
+ disable_streaming_preaggregations=true,
+ disable_join_reorder=true)*/
+ b.k1 AS field1
+ FROM ls_serial a
+ RIGHT OUTER JOIN ls_serial b ON a.v1 > b.v1
+ GROUP BY field1
+ ORDER BY field1 ASC""")
+
+ // GLOBAL_HASH_SHUFFLE fix (Bug 10 from rqg_bugs) — self-join + NLJ with serial exchange
+ checkConsistencyWithSql("rqg_global_hash_shuffle_self_join_nlj",
+ """SELECT /*+SET_VAR(use_serial_exchange=true, parallel_pipeline_task_num=4,
+ ignore_storage_data_distribution=true,
+ disable_join_reorder=true, disable_colocate_plan=true)*/
+ a.k1 AS field1, a.v1 AS field2
+ FROM ls_t1 a
+ LEFT JOIN ls_t1 b ON a.k1 = b.k2
+ LEFT JOIN ls_t1 c ON a.k1 > b.k2
+ WHERE a.v1 > 5
+ GROUP BY field1, field2
+ ORDER BY field1, field2""")
+
+ // FULL OUTER JOIN + GROUP BY with serial exchange (Bug 11 from rqg_bugs)
+ checkConsistencyWithSql("rqg_global_hash_full_outer_join",
+ """SELECT /*+SET_VAR(use_serial_exchange=true, parallel_pipeline_task_num=4,
+ ignore_storage_data_distribution=true)*/
+ a.k1, b.k1, count(1)
+ FROM ls_t1 a
+ FULL OUTER JOIN ls_t2 b ON a.k1 = b.k1
+ WHERE b.k1 = 2
+ GROUP BY a.k1, b.k1
+ ORDER BY 1, 2, 3""")
+
+ // Same pattern but simpler: NLJ with subquery + pooling, no GROUPING SETS
+ checkConsistencyWithSql("nlj_subquery_pooling",
+ """SELECT ${svSerialSource} k1, (SELECT COUNT(*) FROM ls_t2)
+ FROM ls_t1
+ ORDER BY k1""")
+
+ // ================================================================
+ // Phase 2: Fetch all profiles and compare results
+ // By now, all queries have been executed and profiles are being
+ // collected in the background. Fetching them here overlaps the
+ // collection time with query execution, significantly reducing
+ // total test duration.
... 2571 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/07: [test](local shuffle) Add regression test for enable_local_exchange_before_agg
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b35cd3097728072ddb6789ea96cc63851ef67fbe
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue May 12 18:59:27 2026 +0800
[test](local shuffle) Add regression test for enable_local_exchange_before_agg
Covers apache/doris#62438. Tests 10 agg query patterns with both
enable_local_exchange_before_agg=true (FE vs BE planned LE) and
enable_local_exchange_before_agg=false (verify no crash/hang).
---
.../test_enable_local_exchange_before_agg.groovy | 117 +++++++++++++++++++++
1 file changed, 117 insertions(+)
diff --git a/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy b/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy
new file mode 100644
index 00000000000..9871de7484c
--- /dev/null
+++ b/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/**
+ * Regression test for enable_local_exchange_before_agg (apache/doris#62438).
+ *
+ * When enable_local_exchange_before_agg=true (default), BE inserts HASH local exchange
+ * before pre-agg operators. When enable_local_shuffle_planner=true, FE does the same.
+ * This test verifies correctness under both combinations.
+ */
+suite("test_enable_local_exchange_before_agg", "p0") {
+
+ sql "DROP TABLE IF EXISTS le_agg_t1"
+ sql """
+ CREATE TABLE le_agg_t1 (
+ k1 INT NOT NULL,
+ k2 VARCHAR(32),
+ v1 INT,
+ v2 BIGINT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 8
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ sql """INSERT INTO le_agg_t1 VALUES
+ (1, 'a', 10, 100), (1, 'a', 20, 200), (1, 'b', 30, 300),
+ (2, 'a', 40, 400), (2, 'b', 50, 500), (2, 'b', 60, 600),
+ (3, 'c', 70, 700), (3, 'c', 80, 800), (3, 'd', 90, 900),
+ (4, 'a', 100, 1000), (4, 'd', 110, 1100), (4, 'd', 120, 1200),
+ (5, 'e', 130, 1300), (5, 'e', 140, 1400), (5, 'f', 150, 1500)
+ """
+
+ def PH = "PLACEHOLDER"
+
+ def beHints = """/*+SET_VAR(
+ parallel_pipeline_task_num=4,
+ enable_local_shuffle=true,
+ ignore_storage_data_distribution=true,
+ enable_local_shuffle_planner=false,
+ enable_local_exchange_before_agg=true
+ )*/"""
+
+ def feHints = """/*+SET_VAR(
+ parallel_pipeline_task_num=4,
+ enable_local_shuffle=true,
+ ignore_storage_data_distribution=true,
+ enable_local_shuffle_planner=true,
+ enable_local_exchange_before_agg=true
+ )*/"""
+
+ def noLeBeforeAggHints = """/*+SET_VAR(
+ parallel_pipeline_task_num=4,
+ enable_local_shuffle=true,
+ ignore_storage_data_distribution=true,
+ enable_local_shuffle_planner=true,
+ enable_local_exchange_before_agg=false
+ )*/"""
+
+ def queries = [
+ "simple_agg_bucket_key":
+ "SELECT ${PH} k1, SUM(v1), COUNT(*) FROM le_agg_t1 GROUP BY k1 ORDER BY k1",
+ "simple_agg_non_bucket_key":
+ "SELECT ${PH} k2, SUM(v1), MAX(v2) FROM le_agg_t1 GROUP BY k2 ORDER BY k2",
+ "multi_key_agg":
+ "SELECT ${PH} k1, k2, SUM(v1) FROM le_agg_t1 GROUP BY k1, k2 ORDER BY k1, k2",
+ "distinct_non_bucket":
+ "SELECT ${PH} DISTINCT k2 FROM le_agg_t1 ORDER BY k2",
+ "count_distinct":
+ "SELECT ${PH} k1, COUNT(DISTINCT k2), SUM(v1) FROM le_agg_t1 GROUP BY k1 ORDER BY k1",
+ "agg_having":
+ "SELECT ${PH} k2, SUM(v1) AS s FROM le_agg_t1 GROUP BY k2 HAVING SUM(v1) > 100 ORDER BY k2",
+ "agg_after_join":
+ "SELECT ${PH} a.k2, SUM(a.v1) FROM le_agg_t1 a JOIN le_agg_t1 b ON a.k1 = b.k1 GROUP BY a.k2 ORDER BY a.k2",
+ "grouping_sets":
+ "SELECT ${PH} k1, k2, SUM(v1) FROM le_agg_t1 GROUP BY GROUPING SETS ((k1), (k2), (k1, k2)) ORDER BY k1, k2",
+ "window_over_agg":
+ "SELECT ${PH} k1, s, SUM(s) OVER (ORDER BY k1) AS running FROM (SELECT k1, SUM(v1) AS s FROM le_agg_t1 GROUP BY k1) t ORDER BY k1",
+ "multi_distinct":
+ "SELECT ${PH} COUNT(DISTINCT k1), COUNT(DISTINCT k2) FROM le_agg_t1",
+ ]
+
+ // Part 1: FE-planned (enable_local_exchange_before_agg=true) vs BE-planned baseline
+ logger.info("=== Part 1: FE vs BE with enable_local_exchange_before_agg=true ===")
+ queries.each { name, template ->
+ def beResult = sql(template.replace(PH, beHints))
+ def feResult = sql(template.replace(PH, feHints))
+ assertEquals(beResult, feResult, "[${name}] FE-planned differs from BE-planned")
+ logger.info("[${name}] PASSED")
+ }
+
+ // Part 2: enable_local_exchange_before_agg=false — verify no crash/hang and correct results
+ logger.info("=== Part 2: enable_local_exchange_before_agg=false ===")
+ queries.each { name, template ->
+ def beResult = sql(template.replace(PH, beHints))
+ def noLeResult = sql(template.replace(PH, noLeBeforeAggHints))
+ assertEquals(beResult, noLeResult, "[${name}] enable_local_exchange_before_agg=false differs from baseline")
+ logger.info("[${name}] enable_local_exchange_before_agg=false PASSED")
+ }
+
+ sql "DROP TABLE IF EXISTS le_agg_t1"
+ logger.info("=== All enable_local_exchange_before_agg tests completed ===")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/07: [fix](build) Make multi_version.h compile on both libc++ and libstdc++
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 865792164128a8221f604cc41915f4e5cbf14e6e
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed May 13 01:26:39 2026 +0800
[fix](build) Make multi_version.h compile on both libc++ and libstdc++
The previous fix in 35c69241bb7 switched MultiVersion to the free
std::atomic_load / std::atomic_store functions because libc++ in
LLVM 20 (macOS) does not yet specialize std::atomic for
std::shared_ptr (shared_ptr is not trivially_copyable).
But libstdc++ 12+ (used by the Linux RQG compile, gcc 15) marks
those free atomic_load/atomic_store overloads as deprecated in C++20.
Doris builds with -Werror, so the RQG compile fails:
error: 'atomic_load<const doris::RuntimeProfile>' is deprecated:
use 'std::atomic<std::shared_ptr<T>>' instead
[-Werror,-Wdeprecated-declarations]
Drop both atomic paths and use std::shared_mutex:
- shared_lock on get() lets multiple readers proceed concurrently,
matching MultiVersion's "many readers, occasional writer" intent.
- unique_lock on set() serializes the version swap.
- Portable across all toolchains we build with.
MultiVersion is only used on low-frequency paths (metrics scrape /
debug strings / symbol resolution), so the lock overhead vs.
atomic<shared_ptr> is negligible in practice.
---
be/src/common/multi_version.h | 23 +++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/be/src/common/multi_version.h b/be/src/common/multi_version.h
index aadd2ef3237..c3858da779f 100644
--- a/be/src/common/multi_version.h
+++ b/be/src/common/multi_version.h
@@ -21,6 +21,7 @@
#pragma once
#include <memory>
+#include <shared_mutex>
/** Allow to store and read-only usage of an object in several threads,
* and to atomically replace an object in another thread.
@@ -47,20 +48,30 @@ public:
/// Default initialization - by nullptr.
MultiVersion() = default;
- explicit MultiVersion(std::unique_ptr<const T>&& value) : current_version(std::move(value)) {}
+ explicit MultiVersion(std::unique_ptr<const T>&& value)
+ : current_version(Version {std::move(value)}) {}
- // Use the pre-C++20 atomic free functions for shared_ptr instead of
- // std::atomic<shared_ptr>, which requires a trivially_copyable specialization
- // not available in all libc++ versions (e.g., LLVM 20).
+ // Use a shared_mutex (read-write lock) rather than the C++20
+ // std::atomic<std::shared_ptr<T>> specialization or the deprecated free
+ // atomic_load/atomic_store overloads — neither is portable across the
+ // toolchains we build with (libc++ in LLVM 20 lacks the atomic<shared_ptr>
+ // specialization; libstdc++ 12+ marks the free functions deprecated).
+ // MultiVersion is only used on low-frequency paths (metrics scrape,
+ // debug strings, symbol resolution), so the lock overhead is negligible.
/// Obtain current version for read-only usage. Returns shared_ptr, that manages lifetime of version.
- Version get() const { return std::atomic_load(¤t_version); }
+ Version get() const {
+ std::shared_lock lock(mutex);
+ return current_version;
+ }
/// Update an object with new version.
void set(std::unique_ptr<const T>&& value) {
- std::atomic_store(¤t_version, Version {std::move(value)});
+ std::unique_lock lock(mutex);
+ current_version = Version {std::move(value)};
}
private:
+ mutable std::shared_mutex mutex;
Version current_version;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/07: [fix](local shuffle) Phase-aware HASH requirement for AggregationNode
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ddbfca6a3297b790fc4c0bf7876ba2010433f97e
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue May 12 23:09:39 2026 +0800
[fix](local shuffle) Phase-aware HASH requirement for AggregationNode
PR #62438 introduced a semantic split for required_data_distribution=HASH:
- "I'm a shuffled operator, downstream depends on hash for correctness"
(finalize AggSink, HashJoin partitioned, Intersect/Except)
- "I'd like hash distribution for performance optimization"
(StreamingAgg pre-agg with enable_local_exchange_before_agg=true)
Old FE UnionNode used parentRequire.preferType().isHashShuffle() as the
trigger to propagate hash to its inputs. This conflated the two intents —
once StreamingAgg started requesting HASH (PR #62438), every union under
a streaming preagg over-inserted HASH LE on each input branch.
Mirror BE's _followed_by_shuffled_operator propagation:
- PlanNode.requiresShuffleForCorrectness() — default false, override in
AggregationNode (finalize+keys), HashJoinNode (partitioned/bucket-shuffle/
colocate), IntersectNode/ExceptNode (always).
- PlanTranslatorContext.shuffledAncestorMap — per-node flag, set by parent
in enforceRequire before recursing.
- SetOperationNode reads the flag instead of parentRequire type.
Refine the AggregationNode branches with phase-aware HASH requirement (DORIS-25413):
BE's three agg operators all guard with `!_needs_finalize &&
!enable_local_exchange_before_agg → base`, but the semantics differ:
- AggSink: `!finalize && hasKeys` covers BOTH LOCAL preagg (FIRST / SECOND
distinct local, performance-only) AND FIRST_MERGE dedup (correctness).
BE's flag-gated early return wrongly catches FIRST_MERGE, leading to
PASSTHROUGH-over-serial-child → wrong result.
- DistinctStreaming: `!finalize` covers BOTH streaming preagg
(useStreamingPreagg=true, performance) AND non-streaming dedup
(useStreamingPreagg=false, correctness). Same family of bug.
- StreamingAgg: only ever streaming preagg, flag-gated path is fine.
FE fix:
- AggSink: restrict the flag-gated base path to non-merge LOCAL phases
(`!isMerge()`). FIRST_MERGE always goes to HASH regardless of the flag.
- DistinctStreaming: restrict to streaming preagg (`useStreamingPreagg=true`).
Non-streaming dedup always goes to HASH.
- requiresShuffleForCorrectness() mirrors BE's is_shuffled_operator() exactly
(`finalize && hasKeys`). For multi-phase chains, the merge phases above
propagate the flag down via inheritedShuffled — no need to widen here.
Reduces FE/BE consistency mismatches from 4 to 3 (only pre-existing NLJ
optimization differences remain).
---
.../glue/translator/PlanTranslatorContext.java | 15 ++++++++
.../org/apache/doris/planner/AggregationNode.java | 43 +++++++++++++++++++---
.../java/org/apache/doris/planner/ExceptNode.java | 6 +++
.../org/apache/doris/planner/HashJoinNode.java | 9 +++++
.../org/apache/doris/planner/IntersectNode.java | 6 +++
.../java/org/apache/doris/planner/PlanNode.java | 32 ++++++++++++++++
.../org/apache/doris/planner/SetOperationNode.java | 13 ++++---
.../planner/LocalShuffleNodeCoverageTest.java | 5 ++-
.../test_enable_local_exchange_before_agg.groovy | 41 +++++++++++++++++++++
9 files changed, 158 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index db9c8964cec..a0e6fd124c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -111,6 +111,13 @@ public class PlanTranslatorContext {
private final Map<PlanNodeId, Boolean> serialAncestorInPipelineMap = Maps.newHashMap();
+ // Whether any ancestor (or this node itself) requires shuffle for correctness, with
+ // a HASH or NOOP path connecting them. Mirrors BE's _followed_by_shuffled_operator
+ // propagation in pipeline_fragment_context.cpp. UnionNode reads this to decide whether
+ // to propagate hash requirement to its inputs (only when downstream needs shuffle for
+ // correctness, not just for performance like StreamingAgg pre-agg).
+ private final Map<PlanNodeId, Boolean> shuffledAncestorMap = Maps.newHashMap();
+
// Whether the current fragment uses LocalShuffleAssignedJob (pooling scan with
// ignoreDataDistribution → _parallel_instances=1 in BE). When true, serial operators
// indicate real pipeline bottlenecks needing PASSTHROUGH fan-out (heavy_ops).
@@ -252,6 +259,14 @@ public class PlanTranslatorContext {
return serialAncestorInPipelineMap.getOrDefault(node.getId(), false);
}
+ public void setHasShuffleForCorrectnessAncestor(PlanNode node, boolean value) {
+ shuffledAncestorMap.put(node.getId(), value);
+ }
+
+ public boolean hasShuffleForCorrectnessAncestor(PlanNode node) {
+ return shuffledAncestorMap.getOrDefault(node.getId(), false);
+ }
+
public SlotDescriptor addSlotDesc(TupleDescriptor t) {
return descTable.addSlotDescriptor(t);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index ca364960ead..35463dbf080 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -283,10 +283,17 @@ public class AggregationNode extends PlanNode {
// check order 1:1. The helper baseClassRequire() expands BE's base class behavior.
LocalExchangeTypeRequire requireChild;
if (canUseDistinctStreamingAgg(sessionVariable)) {
- // DistinctStreamingAggOperatorX
+ // DistinctStreamingAggOperatorX. Two flavors share this operator class:
+ // - streaming preagg (useStreamingPreagg=true): performance-only,
+ // flag controls
+ // - non-streaming dedup (useStreamingPreagg=false): correctness-required,
+ // always HASH regardless of flag
+ // Diverges from BE: BE's `!_needs_finalize && !enable_local_exchange_before_agg`
+ // early return catches non-streaming dedup too, causing the same family of
+ // wrong-result bug as AggSink (DORIS-25413).
if (needsFinalize && !hasKeys) {
requireChild = LocalExchangeTypeRequire.noRequire();
- } else if (!needsFinalize && !enableLeBeforeAgg) {
+ } else if (!needsFinalize && useStreamingPreagg && !enableLeBeforeAgg) {
requireChild = baseClassRequire(connectContext);
} else if (needsFinalize || (hasKeys && !useStreamingPreagg)) {
requireChild = AddLocalExchange.isColocated(this)
@@ -312,15 +319,29 @@ public class AggregationNode extends PlanNode {
requireChild = LocalExchangeTypeRequire.requireHash();
}
} else {
- // AggSinkOperatorX
+ // AggSinkOperatorX — covers finalize phase AND non-finalize phases (LOCAL
+ // preagg / FIRST_MERGE dedup). Streaming preagg goes through the StreamingAgg
+ // branch above, not here.
+ //
+ // Phase semantics for !needsFinalize:
+ // - FIRST / SECOND (LOCAL phase, !isMerge): performance-only, flag controls
+ // - FIRST_MERGE (correctness-required): always HASH regardless of flag
+ //
+ // Diverges from BE here: BE's `!_needs_finalize && !enable_local_exchange_before_agg`
+ // early return also catches FIRST_MERGE, dropping the HASH requirement and
+ // causing wrong-result (e.g. PASSTHROUGH over serial child breaks the
+ // group-by-key invariant — DORIS-25413).
if (!hasKeys) {
requireChild = needsFinalize
? LocalExchangeTypeRequire.noRequire()
: baseClassRequire(connectContext);
- } else if (!needsFinalize && !enableLeBeforeAgg) {
+ } else if (!needsFinalize && !aggInfo.isMerge() && !enableLeBeforeAgg) {
+ // LOCAL phase (FIRST preagg / SECOND distinct local) + user opted out
+ // of pre-agg LE → base class decides: serial child → PASSTHROUGH
+ // (parallelism), non-serial child → NOOP (no LE).
requireChild = baseClassRequire(connectContext);
} else if (!needsFinalize || AddLocalExchange.isColocated(this)) {
- // BE: non-finalize (PR #62438) or colocate → HASH
+ // FIRST_MERGE (correctness) or finalize+colocate → HASH.
requireChild = parentRequire.autoRequireHash();
} else if (hasPartitionExprs(parentRequire)) {
// FE-only heuristic: finalize non-colocate with parent hash requirement
@@ -351,6 +372,18 @@ public class AggregationNode extends PlanNode {
return aggInfo.getGroupingExprs();
}
+ @Override
+ public boolean requiresShuffleForCorrectness() {
+ // Mirrors BE's AggSinkOperatorX::is_shuffled_operator() exactly:
+ // finalize agg with group keys needs hash-distributed input for correctness.
+ // GLOBAL dedup (!needsFinalize) is intentionally NOT included here — if a
+ // GLOBAL dedup exists, a finalize agg always sits above it (e.g. DISTINCT_GLOBAL
+ // above DISTINCT_LOCAL/GLOBAL_DEDUP), and the finalize agg propagates the flag
+ // down via inheritedShuffled. A solo finalize agg satisfies hash distribution
+ // through its own child requirement.
+ return needsFinalize && !aggInfo.getGroupingExprs().isEmpty();
+ }
+
private boolean canUseDistinctStreamingAgg(SessionVariable sessionVariable) {
return aggInfo.getAggregateExprs().isEmpty() && sortByGroupKey == null
&& sessionVariable.enableDistinctStreamingAggregation;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java
index ae728da571b..1052b6bd118 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java
@@ -30,4 +30,10 @@ public class ExceptNode extends SetOperationNode {
protected void toThrift(TPlanNode msg) {
toThrift(msg, TPlanNodeType.EXCEPT_NODE);
}
+
+ @Override
+ public boolean requiresShuffleForCorrectness() {
+ // BE: SetSinkOperatorX / SetSourceOperatorX.is_shuffled_operator() = true (unconditional).
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index 87c3094a361..ae01b515cab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -132,6 +132,15 @@ public class HashJoinNode extends JoinNodeBase {
return isColocate;
}
+ @Override
+ public boolean requiresShuffleForCorrectness() {
+ // BE: HashJoinBuild/Probe.is_shuffled_operator() = PARTITIONED || BUCKET_SHUFFLE || COLOCATE.
+ // (BROADCAST and NONE are not shuffled — they don't depend on hash distribution.)
+ return distrMode == DistributionMode.PARTITIONED
+ || distrMode == DistributionMode.BUCKET_SHUFFLE
+ || isColocate;
+ }
+
public Map<ExprId, SlotId> getHashOutputExprSlotIdMap() {
return hashOutputExprSlotIdMap;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java
index 89f4002ea4d..5d784c03152 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java
@@ -30,4 +30,10 @@ public class IntersectNode extends SetOperationNode {
protected void toThrift(TPlanNode msg) {
toThrift(msg, TPlanNodeType.INTERSECT_NODE);
}
+
+ @Override
+ public boolean requiresShuffleForCorrectness() {
+ // BE: SetSinkOperatorX / SetSourceOperatorX.is_shuffled_operator() = true (unconditional).
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 3a2438a85dd..c3d38b89bd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -840,6 +840,23 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
return fragment != null && isSerialNode() && fragment.useSerialSource(context);
}
+ /**
+ * "I depend on hash distribution for correctness, not just performance optimization."
+ * Used by UnionNode to decide whether to propagate hash requirement to its inputs:
+ * when a downstream operator requires shuffle for correctness, Union must pre-shuffle
+ * its inputs so the merged output is hash-distributed.
+ *
+ * Default is false; only operators that truly need hash for correctness override
+ * (finalize AggSink with group keys, HashJoin PARTITIONED/BUCKET_SHUFFLE, Intersect,
+ * Except). Operators that request hash for performance only (StreamingAgg pre-agg
+ * with enable_local_exchange_before_agg) MUST NOT override.
+ *
+ * Mirrors BE's OperatorBase::is_shuffled_operator().
+ */
+ public boolean requiresShuffleForCorrectness() {
+ return false;
+ }
+
public boolean hasSerialChildren() {
if (children.isEmpty()) {
return isSerialNode();
@@ -980,6 +997,21 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
boolean childHasSerialAncestor = inheritedSerial || isSerialNode();
translatorContext.setHasSerialAncestorInPipeline(child, childHasSerialAncestor);
+ // 1b. Propagate shuffle-for-correctness-ancestor flag to child.
+ // Mirrors BE's _followed_by_shuffled_operator: a downstream operator needs hash
+ // distribution for correctness, and the chain to here goes through HASH or NOOP
+ // requirements (so the dependency is preserved).
+ // propagate = ((inheritedShuffled || self.requiresShuffleForCorrectness)
+ // && require is hash)
+ // || (inheritedShuffled && require is noop/passthrough)
+ boolean inheritedShuffled = translatorContext.hasShuffleForCorrectnessAncestor(this);
+ boolean selfOrInheritedShuffled = inheritedShuffled || requiresShuffleForCorrectness();
+ boolean requireIsHash = require.preferType().isHashShuffle();
+ boolean requireIsNoop = require.preferType() == LocalExchangeNode.LocalExchangeType.NOOP;
+ boolean childShuffledAncestor = (selfOrInheritedShuffled && requireIsHash)
+ || (inheritedShuffled && requireIsNoop);
+ translatorContext.setHasShuffleForCorrectnessAncestor(child, childShuffledAncestor);
+
// 2. Recurse child (Layer 2: child declares its own require/output)
Pair<PlanNode, LocalExchangeType> childOutput =
child.enforceAndDeriveLocalExchange(translatorContext, this, require);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
index a7d89278f0c..1c7d399b2b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
@@ -210,11 +210,14 @@ public abstract class SetOperationNode extends PlanNode {
LocalExchangeType outputType;
PlanNode firstChild = children.isEmpty() ? null : children.get(0);
if (this instanceof UnionNode) {
- // Propagate parent's hash requirement to children when parent requires hash distribution.
- // Matches BE's UnionSinkOperatorX which returns GLOBAL_HASH(_distribute_exprs) whenever
- // _followed_by_shuffled_operator=true, regardless of whether _distribute_exprs is empty.
- boolean canPropagateHash = parentRequire.preferType().isHashShuffle();
- requireChild = canPropagateHash ? parentRequire : LocalExchangeTypeRequire.noRequire();
+ // Propagate parent's hash requirement to children ONLY when a downstream operator
+ // requires shuffle for correctness (not just performance optimization). Matches BE's
+ // UnionSinkOperatorX which returns GLOBAL_HASH(_distribute_exprs) whenever
+ // _followed_by_shuffled_operator=true. The flag is propagated by enforceRequire
+ // from operators with requiresShuffleForCorrectness()=true (finalize agg, hash join,
+ // intersect/except) through hash/noop links.
+ boolean canPropagateHash = translatorContext.hasShuffleForCorrectnessAncestor(this);
+ requireChild = canPropagateHash ? parentRequire.autoRequireHash() : LocalExchangeTypeRequire.noRequire();
outputType = canPropagateHash
? AddLocalExchange.resolveExchangeType(requireChild, translatorContext, this, firstChild)
: LocalExchangeType.NOOP;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
index 9f3fea43869..4d86a4aacc0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java
@@ -348,8 +348,9 @@ public class LocalShuffleNodeCoverageTest {
UnionNode unionNode = new UnionNode(nextPlanNodeId(), new TupleId(NEXT_ID.getAndIncrement()));
TrackingPlanNode unionChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP);
unionNode.addChild(unionChild);
- // UnionNode propagates parent hash require to children when parent requires hash.
- // resolveExchangeType with RequireHash → LOCAL_EXECUTION_HASH_SHUFFLE
+ // UnionNode propagates parent hash require to children only when a downstream operator
+ // requires shuffle for correctness. Simulate that via the context flag.
+ ctx.setHasShuffleForCorrectnessAncestor(unionNode, true);
Pair<PlanNode, LocalExchangeType> unionOutput = unionNode.enforceAndDeriveLocalExchange(
ctx, null, LocalExchangeTypeRequire.requireHash());
Assertions.assertEquals(LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, unionOutput.second);
diff --git a/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy b/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy
index 9871de7484c..77500d54f82 100644
--- a/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy
+++ b/regression-test/suites/nereids_p0/local_shuffle/test_enable_local_exchange_before_agg.groovy
@@ -71,6 +71,31 @@ suite("test_enable_local_exchange_before_agg", "p0") {
enable_local_exchange_before_agg=false
)*/"""
+ // Forces LOCAL preagg through the non-streaming AggSink / DistinctStreaming paths
+ // (instead of the StreamingAgg branch). Combined with enable_local_exchange_before_agg=false,
+ // this exercises the phase-aware fix:
+ // - AggSink: !isMerge() LOCAL phase → base (PASSTHROUGH/NOOP)
+ // FIRST_MERGE → HASH (correctness, regardless of flag)
+ // - DistinctStreaming: useStreamingPreagg=true → base
+ // useStreamingPreagg=false → HASH (correctness, regardless of flag)
+ def disableStreamingHints = """/*+SET_VAR(
+ parallel_pipeline_task_num=4,
+ enable_local_shuffle=true,
+ ignore_storage_data_distribution=true,
+ enable_local_shuffle_planner=true,
+ enable_local_exchange_before_agg=false,
+ disable_streaming_preaggregations=true
+ )*/"""
+
+ def disableStreamingFlagOnHints = """/*+SET_VAR(
+ parallel_pipeline_task_num=4,
+ enable_local_shuffle=true,
+ ignore_storage_data_distribution=true,
+ enable_local_shuffle_planner=true,
+ enable_local_exchange_before_agg=true,
+ disable_streaming_preaggregations=true
+ )*/"""
+
def queries = [
"simple_agg_bucket_key":
"SELECT ${PH} k1, SUM(v1), COUNT(*) FROM le_agg_t1 GROUP BY k1 ORDER BY k1",
@@ -112,6 +137,22 @@ suite("test_enable_local_exchange_before_agg", "p0") {
logger.info("[${name}] enable_local_exchange_before_agg=false PASSED")
}
+ // Part 3: disable_streaming_preaggregations=true — forces AggSink / DistinctStreaming
+ // non-streaming paths. Combined with the two flag values, exercises the phase-aware
+ // fix on both LOCAL (performance, flag controls) and MERGE/non-streaming-dedup
+ // (correctness, always HASH) sub-paths.
+ logger.info("=== Part 3: disable_streaming_preaggregations=true ===")
+ queries.each { name, template ->
+ def beResult = sql(template.replace(PH, beHints))
+ def disableStreamingFlagOnResult = sql(template.replace(PH, disableStreamingFlagOnHints))
+ assertEquals(beResult, disableStreamingFlagOnResult,
+ "[${name}] disable_streaming+flag=true differs from baseline")
+ def disableStreamingResult = sql(template.replace(PH, disableStreamingHints))
+ assertEquals(beResult, disableStreamingResult,
+ "[${name}] disable_streaming+flag=false differs from baseline")
+ logger.info("[${name}] disable_streaming both flag values PASSED")
+ }
+
sql "DROP TABLE IF EXISTS le_agg_t1"
logger.info("=== All enable_local_exchange_before_agg tests completed ===")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/07: [fix](build) Add missing include in multi_version.h
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch fe_local_shuffle_rebase2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6d44b28c3e50a911a8b57a1c48f7a89b102c776f
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed May 13 09:25:29 2026 +0800
[fix](build) Add missing <mutex> include in multi_version.h
std::unique_lock lives in <mutex>, not <shared_mutex>. libc++ (macOS)
transitively pulls it in via <shared_mutex>, so the local build passed;
libstdc++ on the Linux RQG compile is stricter and fails:
error: no member named 'unique_lock' in namespace 'std'
Add the explicit include to keep both toolchains happy.
---
be/src/common/multi_version.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/be/src/common/multi_version.h b/be/src/common/multi_version.h
index c3858da779f..b1c82313ea5 100644
--- a/be/src/common/multi_version.h
+++ b/be/src/common/multi_version.h
@@ -21,6 +21,7 @@
#pragma once
#include <memory>
+#include <mutex>
#include <shared_mutex>
/** Allow to store and read-only usage of an object in several threads,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org