You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by co...@apache.org on 2026/06/12 22:22:46 UTC
(datafusion-comet) branch main updated: fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)
This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new f6738d653d fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)
f6738d653d is described below
commit f6738d653d594cfe1b1b47a5f1c445a93f3f4960
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri Jun 12 16:22:43 2026 -0600
fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)
The native BNLJ rule from #4429 wraps BroadcastNestedLoopJoinExec for
all Spark profiles, but only 3.5.8/4.0.2/4.1.2 diffs were updated to
accept the Comet variant in JoinSuite's SPARK-34593 plan-shape checks.
Mirror the same change in the 3.4.3 diff so tests 1 and 3 (the cases
where Comet's getSupportLevel returns Compatible) match either class.
---
dev/diffs/3.4.3.diff | 35 ++++++++++++++++++++++++++---------
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index 170cd2f190..937a57b79b 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -693,7 +693,7 @@ index 1792b4c32eb..1616e6f39bd 100644
assert(shuffleMergeJoins.size == 1)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-index 7f062bfb899..0ed85486e80 100644
+index 7f062bfb899..5b2fa3caf65 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
@@ -30,7 +30,8 @@ import org.apache.spark.sql.catalyst.TableIdentifier
@@ -818,7 +818,19 @@ index 7f062bfb899..0ed85486e80 100644
checkAnswer(shjNonCodegenDF, Seq.empty)
}
}
-@@ -1341,7 +1359,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1325,7 +1343,10 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+ "/*+ BROADCAST(t2) */ t1.k as k"
+ }
+ val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
+- assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
++ assert(collect(plan) {
++ case _: BroadcastNestedLoopJoinExec | _: CometBroadcastNestedLoopJoinExec =>
++ true
++ }.size === 1)
+ // No extra shuffle before aggregation
+ assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 0)
+ }
+@@ -1341,7 +1362,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
// Have shuffle before aggregation
@@ -828,12 +840,17 @@ index 7f062bfb899..0ed85486e80 100644
}
def getJoinQuery(selectExpr: String, joinType: String): String = {
-@@ -1370,9 +1389,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1369,10 +1391,16 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+ "/*+ BROADCAST(right_t) */ k1 as k0"
}
val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
- assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+- assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
- assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
+ assert(collect(plan) {
++ case _: BroadcastNestedLoopJoinExec | _: CometBroadcastNestedLoopJoinExec =>
++ true
++ }.size === 1)
++ assert(collect(plan) {
+ case _: SortMergeJoinExec => true
+ case _: CometSortMergeJoinExec => true
+ }.size === 3)
@@ -843,7 +860,7 @@ index 7f062bfb899..0ed85486e80 100644
}
// Test output ordering is not preserved
-@@ -1381,9 +1403,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1381,9 +1409,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
val selectExpr = "/*+ BROADCAST(left_t) */ k1 as k0"
val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
@@ -858,7 +875,7 @@ index 7f062bfb899..0ed85486e80 100644
}
// Test singe partition
-@@ -1393,7 +1418,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1393,7 +1424,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
|FROM range(0, 10, 1, 1) t1 FULL OUTER JOIN range(0, 10, 1, 1) t2
|""".stripMargin)
val plan = fullJoinDF.queryExecution.executedPlan
@@ -868,7 +885,7 @@ index 7f062bfb899..0ed85486e80 100644
checkAnswer(fullJoinDF, Row(100))
}
}
-@@ -1438,6 +1464,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1438,6 +1470,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
Seq(semiJoinDF, antiJoinDF).foreach { df =>
assert(collect(df.queryExecution.executedPlan) {
case j: ShuffledHashJoinExec if j.ignoreDuplicatedKey == ignoreDuplicatedKey => true
@@ -878,7 +895,7 @@ index 7f062bfb899..0ed85486e80 100644
}.size == 1)
}
}
-@@ -1482,14 +1511,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1482,14 +1517,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") {
def check(plan: SparkPlan): Unit = {
@@ -901,7 +918,7 @@ index 7f062bfb899..0ed85486e80 100644
}
dupStreamSideColTest("SHUFFLE_HASH", check)
}
-@@ -1605,7 +1640,8 @@ class ThreadLeakInSortMergeJoinSuite
+@@ -1605,7 +1646,8 @@ class ThreadLeakInSortMergeJoinSuite
sparkConf.set(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD, 20))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org