You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/29 23:14:17 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6280-674790919bd734c9b55c2cbb4018ef93a376de7c created (now 098f58db88)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6280-674790919bd734c9b55c2cbb4018ef93a376de7c
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 098f58db88 test: check Iceberg DPP pruning by planned file tasks (#6280)
This branch includes the following new commits:
new 098f58db88 test: check Iceberg DPP pruning by planned file tasks (#6280)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: test: check Iceberg DPP pruning by planned file tasks (#6280)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6280-674790919bd734c9b55c2cbb4018ef93a376de7c
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 098f58db88a250ab79c48575983428ce0c704c9f
Author: dustin <dw...@users.noreply.github.com>
AuthorDate: Tue Sep 29 20:53:00 2026 +0000
test: check Iceberg DPP pruning by planned file tasks (#6280)
Two Iceberg DPP tests used numPartitions == 1 as evidence of pruning, but
Iceberg packs their small files into one Spark partition whether or not DPP
prunes, so the checks passed with DPP off. Count the file tasks the scan
planned instead, and the splits read where the scan runs once.
---
.../org/apache/comet/CometIcebergNativeSuite.scala | 26 +++++++++++++++++-----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
index e156bec9c2..2882e778bc 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
@@ -3837,8 +3837,16 @@ class CometIcebergNativeSuite
assert(
icebergScans.nonEmpty,
s"Expected CometIcebergNativeScanExec but found none. Plan:\n$cometPlan")
- val numPartitions = icebergScans.head.numPartitions
- assert(numPartitions == 1, s"Expected DPP to prune to 1 partition but got $numPartitions")
+ // Iceberg packs these small files into one Spark partition whether or not DPP prunes, so
+ // count the planned file tasks instead: 1 of the 3 date partitions' files is left. The
+ // num_splits metric is not used here because the ORDER BY's range partitioning runs the
+ // scan once to sample bounds and again for the shuffle, so each split is read twice.
+ val plannedTasks = icebergScans.head.perPartitionData
+ .map(OperatorOuterClass.IcebergScan.parseFrom(_).getFileScanTasksCount)
+ .sum
+ assert(
+ plannedTasks == 1,
+ s"Expected DPP to prune to 1 of 3 files but planned $plannedTasks tasks:\n$cometPlan")
// Verify AQE DPP used CometSubqueryBroadcastExec with broadcast reuse
if (isSpark35Plus) {
@@ -4485,12 +4493,20 @@ class CometIcebergNativeSuite
// reuse manifests as ReusedExchangeExec inside the ASPE's final plan.
assertCsbBroadcastReuse(subqueries, cometPlan)
- // Verify correct results and partition pruning
+ // Verify partition pruning. Iceberg packs these small files into one Spark partition
+ // whether or not DPP prunes, so count the planned file tasks and the splits read: only
+ // the file for 1970-01-02 of the 3 dates should be left.
val icebergScans = collectIcebergNativeScans(cometPlan)
assert(icebergScans.nonEmpty, "Expected CometIcebergNativeScanExec in plan")
+ val scan = icebergScans.head
+ val plannedTasks = scan.perPartitionData
+ .map(OperatorOuterClass.IcebergScan.parseFrom(_).getFileScanTasksCount)
+ .sum
+ val numSplits = scan.metrics("num_splits").value
assert(
- icebergScans.head.numPartitions == 1,
- s"Expected DPP to prune to 1 partition but got ${icebergScans.head.numPartitions}")
+ plannedTasks == 1 && numSplits == 1,
+ s"Expected DPP to prune to 1 of 3 files, planned $plannedTasks tasks and read " +
+ s"$numSplits splits:\n${cometPlan.treeString}")
}
spark.sql("DROP TABLE aqe_cat.db.dpp_reuse_fact")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org