You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by be...@apache.org on 2025/04/21 08:52:41 UTC
(spark) branch master updated: [SPARK-51663][SQL][FOLLOWUP] change buildLeft and buildRight to function
This is an automated email from the ASF dual-hosted git repository.
beliefer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 929042cfcfa5 [SPARK-51663][SQL][FOLLOWUP] change buildLeft and buildRight to function
929042cfcfa5 is described below
commit 929042cfcfa5eadf8a903b6aafe7027cdb0594e4
Author: beliefer <be...@163.com>
AuthorDate: Mon Apr 21 16:52:27 2025 +0800
[SPARK-51663][SQL][FOLLOWUP] change buildLeft and buildRight to function
### What changes were proposed in this pull request?
This PR follows up https://github.com/apache/spark/pull/50455 and improves `JoinSelectionHelper` more.
### Why are the changes needed?
The overhead of `buildLeft`/`buildRight` are more expensive than `canBuildBroadcastLeft`/`canBuildBroadcastRight`.
### Does this PR introduce _any_ user-facing change?
'No'.
### How was this patch tested?
GA.
### Was this patch authored or co-authored using generative AI tooling?
'No'.
Closes #50636 from beliefer/SPARK-51663_followup.
Authored-by: beliefer <be...@163.com>
Signed-off-by: beliefer <be...@163.com>
---
.../spark/sql/catalyst/optimizer/joins.scala | 60 ++++++++++++----------
1 file changed, 34 insertions(+), 26 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
index 44875bfb3ec2..9413bd7b454d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
@@ -293,19 +293,23 @@ trait JoinSelectionHelper extends Logging {
join: Join,
hintOnly: Boolean,
conf: SQLConf): Option[BuildSide] = {
- val buildLeft = if (hintOnly) {
- hintToBroadcastLeft(join.hint)
- } else {
- canBroadcastBySize(join.left, conf) && !hintToNotBroadcastLeft(join.hint)
+ def shouldBuildLeft(): Boolean = {
+ if (hintOnly) {
+ hintToBroadcastLeft(join.hint)
+ } else {
+ canBroadcastBySize(join.left, conf) && !hintToNotBroadcastLeft(join.hint)
+ }
}
- val buildRight = if (hintOnly) {
- hintToBroadcastRight(join.hint)
- } else {
- canBroadcastBySize(join.right, conf) && !hintToNotBroadcastRight(join.hint)
+ def shouldBuildRight(): Boolean = {
+ if (hintOnly) {
+ hintToBroadcastRight(join.hint)
+ } else {
+ canBroadcastBySize(join.right, conf) && !hintToNotBroadcastRight(join.hint)
+ }
}
getBuildSide(
- buildLeft && canBuildBroadcastLeft(join.joinType),
- buildRight && canBuildBroadcastRight(join.joinType),
+ canBuildBroadcastLeft(join.joinType) && shouldBuildLeft(),
+ canBuildBroadcastRight(join.joinType) && shouldBuildRight(),
join.left,
join.right
)
@@ -315,25 +319,29 @@ trait JoinSelectionHelper extends Logging {
join: Join,
hintOnly: Boolean,
conf: SQLConf): Option[BuildSide] = {
- val buildLeft = if (hintOnly) {
- hintToShuffleHashJoinLeft(join.hint)
- } else {
- hintToPreferShuffleHashJoinLeft(join.hint) ||
- (!conf.preferSortMergeJoin && canBuildLocalHashMapBySize(join.left, conf) &&
- muchSmaller(join.left, join.right, conf)) ||
- forceApplyShuffledHashJoin(conf)
+ def shouldBuildLeft(): Boolean = {
+ if (hintOnly) {
+ hintToShuffleHashJoinLeft(join.hint)
+ } else {
+ hintToPreferShuffleHashJoinLeft(join.hint) ||
+ (!conf.preferSortMergeJoin && canBuildLocalHashMapBySize(join.left, conf) &&
+ muchSmaller(join.left, join.right, conf)) ||
+ forceApplyShuffledHashJoin(conf)
+ }
}
- val buildRight = if (hintOnly) {
- hintToShuffleHashJoinRight(join.hint)
- } else {
- hintToPreferShuffleHashJoinRight(join.hint) ||
- (!conf.preferSortMergeJoin && canBuildLocalHashMapBySize(join.right, conf) &&
- muchSmaller(join.right, join.left, conf)) ||
- forceApplyShuffledHashJoin(conf)
+ def shouldBuildRight(): Boolean = {
+ if (hintOnly) {
+ hintToShuffleHashJoinRight(join.hint)
+ } else {
+ hintToPreferShuffleHashJoinRight(join.hint) ||
+ (!conf.preferSortMergeJoin && canBuildLocalHashMapBySize(join.right, conf) &&
+ muchSmaller(join.right, join.left, conf)) ||
+ forceApplyShuffledHashJoin(conf)
+ }
}
getBuildSide(
- buildLeft && canBuildShuffledHashJoinLeft(join.joinType),
- buildRight && canBuildShuffledHashJoinRight(join.joinType),
+ canBuildShuffledHashJoinLeft(join.joinType) && shouldBuildLeft(),
+ canBuildShuffledHashJoinRight(join.joinType) && shouldBuildRight(),
join.left,
join.right
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org