You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by co...@apache.org on 2026/06/12 22:22:46 UTC

(datafusion-comet) branch main updated: fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)

This is an automated email from the ASF dual-hosted git repository.

comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new f6738d653d fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)
f6738d653d is described below

commit f6738d653d594cfe1b1b47a5f1c445a93f3f4960
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri Jun 12 16:22:43 2026 -0600

    fix: accept CometBroadcastNestedLoopJoinExec in Spark 3.4 SPARK-34593 plan assertions (#4644)
    
    The native BNLJ rule from #4429 wraps BroadcastNestedLoopJoinExec for
    all Spark profiles, but only 3.5.8/4.0.2/4.1.2 diffs were updated to
    accept the Comet variant in JoinSuite's SPARK-34593 plan-shape checks.
    Mirror the same change in the 3.4.3 diff so tests 1 and 3 (the cases
    where Comet's getSupportLevel returns Compatible) match either class.
---
 dev/diffs/3.4.3.diff | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index 170cd2f190..937a57b79b 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -693,7 +693,7 @@ index 1792b4c32eb..1616e6f39bd 100644
      assert(shuffleMergeJoins.size == 1)
    }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-index 7f062bfb899..0ed85486e80 100644
+index 7f062bfb899..5b2fa3caf65 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
 @@ -30,7 +30,8 @@ import org.apache.spark.sql.catalyst.TableIdentifier
@@ -818,7 +818,19 @@ index 7f062bfb899..0ed85486e80 100644
            checkAnswer(shjNonCodegenDF, Seq.empty)
          }
        }
-@@ -1341,7 +1359,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1325,7 +1343,10 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+             "/*+ BROADCAST(t2) */ t1.k as k"
+           }
+           val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
+-          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
++          assert(collect(plan) {
++            case _: BroadcastNestedLoopJoinExec | _: CometBroadcastNestedLoopJoinExec =>
++              true
++          }.size === 1)
+           // No extra shuffle before aggregation
+           assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 0)
+       }
+@@ -1341,7 +1362,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
            val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
            assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
            // Have shuffle before aggregation
@@ -828,12 +840,17 @@ index 7f062bfb899..0ed85486e80 100644
        }
  
        def getJoinQuery(selectExpr: String, joinType: String): String = {
-@@ -1370,9 +1389,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1369,10 +1391,16 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+             "/*+ BROADCAST(right_t) */ k1 as k0"
            }
            val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
+-          assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
 -          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
 +          assert(collect(plan) {
++            case _: BroadcastNestedLoopJoinExec | _: CometBroadcastNestedLoopJoinExec =>
++              true
++          }.size === 1)
++          assert(collect(plan) {
 +            case _: SortMergeJoinExec => true
 +            case _: CometSortMergeJoinExec => true
 +          }.size === 3)
@@ -843,7 +860,7 @@ index 7f062bfb899..0ed85486e80 100644
        }
  
        // Test output ordering is not preserved
-@@ -1381,9 +1403,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1381,9 +1409,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
            val selectExpr = "/*+ BROADCAST(left_t) */ k1 as k0"
            val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
            assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
@@ -858,7 +875,7 @@ index 7f062bfb899..0ed85486e80 100644
        }
  
        // Test singe partition
-@@ -1393,7 +1418,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1393,7 +1424,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
             |FROM range(0, 10, 1, 1) t1 FULL OUTER JOIN range(0, 10, 1, 1) t2
             |""".stripMargin)
        val plan = fullJoinDF.queryExecution.executedPlan
@@ -868,7 +885,7 @@ index 7f062bfb899..0ed85486e80 100644
        checkAnswer(fullJoinDF, Row(100))
      }
    }
-@@ -1438,6 +1464,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1438,6 +1470,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
            Seq(semiJoinDF, antiJoinDF).foreach { df =>
              assert(collect(df.queryExecution.executedPlan) {
                case j: ShuffledHashJoinExec if j.ignoreDuplicatedKey == ignoreDuplicatedKey => true
@@ -878,7 +895,7 @@ index 7f062bfb899..0ed85486e80 100644
              }.size == 1)
            }
        }
-@@ -1482,14 +1511,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1482,14 +1517,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
  
    test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") {
      def check(plan: SparkPlan): Unit = {
@@ -901,7 +918,7 @@ index 7f062bfb899..0ed85486e80 100644
      }
      dupStreamSideColTest("SHUFFLE_HASH", check)
    }
-@@ -1605,7 +1640,8 @@ class ThreadLeakInSortMergeJoinSuite
+@@ -1605,7 +1646,8 @@ class ThreadLeakInSortMergeJoinSuite
        sparkConf.set(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD, 20))
    }
  


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org