You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/12 19:31:13 UTC
(datafusion-comet) 01/01: test: expect Spark 3.5's empty LocalTableScan in the single-split DPP join test (#6902)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6902-c937df09c2cbd302772310a85f14ca52f32455c7
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 42db91b2e892a28965248c8ba84c941eeb6769e8
Author: dustin <dw...@users.noreply.github.com>
AuthorDate: Mon Oct 12 19:30:53 2026 +0000
test: expect Spark 3.5's empty LocalTableScan in the single-split DPP join test (#6902)
With AQE on, once DPP prunes one shuffled input, AQEPropagateEmptyRelation
replaces the join with an empty relation. Spark 4.0 plans that as
EmptyRelationExec, which Comet converts to CometEmptyRelationExec, but
Spark 3.5 has no EmptyRelation node and plans an empty LocalTableScanExec.
Check for the node each version plans.
Closes #6901.
---
.../scala/org/apache/comet/CometIcebergNativeSuite.scala | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
index 4180e3789e..b00f3353e4 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
@@ -36,7 +36,7 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
import org.apache.spark.sql.catalyst.expressions.DynamicPruningExpression
import org.apache.spark.sql.comet._
import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec}
-import org.apache.spark.sql.execution.{InSubqueryExec, ReusedSubqueryExec, SparkPlan, SubqueryAdaptiveBroadcastExec, SubqueryBroadcastExec, SubqueryExec}
+import org.apache.spark.sql.execution.{InSubqueryExec, LocalTableScanExec, ReusedSubqueryExec, SparkPlan, SubqueryAdaptiveBroadcastExec, SubqueryBroadcastExec, SubqueryExec}
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, BroadcastQueryStageExec}
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
@@ -5322,10 +5322,21 @@ class CometIcebergNativeSuite
if (isSpark35Plus && aqeEnabled && scans.isEmpty) {
// Newer Spark versions shuffle the join inputs, allowing AQE to replace
// the join and its scans with an empty relation after DPP prunes one input.
+ // Spark 4.0+ (SPARK-48466) plans it as EmptyRelationExec, which Comet
+ // converts to CometEmptyRelationExec. Spark 3.5 has no EmptyRelation node
+ // and plans an empty LocalTableScanExec instead.
// This checks result compatibility; the SinglePartition regression assertions
// below target Spark 3.4, where both scans must remain in one native block.
+ val emptyRelations = if (isSpark40Plus) {
+ collect(plan) { case empty: CometEmptyRelationExec => empty }
+ } else {
+ collect(plan) {
+ case empty: LocalTableScanExec if empty.rows.isEmpty => empty
+ case empty: CometLocalTableScanExec if empty.rows.isEmpty => empty
+ }
+ }
assert(
- collect(plan) { case empty: CometEmptyRelationExec => empty }.nonEmpty,
+ emptyRelations.nonEmpty,
s"Expected AQE to replace the pruned join with an empty relation:\n$plan")
} else {
assert(scans.size == 2, s"Expected two native Iceberg scans:\n$plan")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org