You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/09 16:39:35 UTC

(datafusion) 01/01: fix: report `LowerEqual` cardinality effect for `CoalescePartitionsExec` with fetch (#26051)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-26051-95bbb263c6fb8f7030823671b4e6fd8ee9f5394d
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit bd3b8c183d97a9a6c2029c98fbf2895d41686174
Author: Alessandro Solimando <as...@apache.org>
AuthorDate: Fri Oct 9 16:39:19 2026 +0000

    fix: report `LowerEqual` cardinality effect for `CoalescePartitionsExec` with fetch (#26051)
    
    ## Which issue does this PR close?
    
    - Closes #26050.
    
    ## Rationale for this change
    
    `CoalescePartitionsExec::cardinality_effect` returns
    `CardinalityEffect::Equal` even when `fetch` is set, although the
    operator can then produce fewer rows than its input. Code that relies on
    `cardinality_effect` gets the wrong answer, for example
    `PassthroughStatisticsProvider` reports the input row count and drops
    the fetch.
    
    ## What changes are included in this PR?
    
    - `CoalescePartitionsExec::cardinality_effect` returns
    `CardinalityEffect::LowerEqual` when `fetch` is set, as `SortExec` and
    `SortPreservingMergeExec` already do.
    
    ## What is the testing strategy for this PR?
    
    - New unit test for `cardinality_effect` with and without `fetch`.
    - New unit test showing that `PassthroughStatisticsProvider` no longer
    drops the fetch (`Exact(10)` instead of `Exact(1000)`).
    - `sqllogictest` and the core `physical_optimizer` integration tests
    pass with no plan changes.
    
    ## Are there any user-facing changes?
    
    No.
    
    ----
    
    Disclaimer: I used AI to assist in the code generation, I have manually
    reviewed the output and it matches my intention and understanding.
---
 .../physical-plan/src/coalesce_partitions.rs       | 23 +++++++++++++++++++++-
 .../physical-plan/src/operator_statistics/mod.rs   | 16 +++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 9e3811e0ad..d3de70a3d9 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -304,7 +304,11 @@ impl ExecutionPlan for CoalescePartitionsExec {
     }
 
     fn cardinality_effect(&self) -> CardinalityEffect {
-        CardinalityEffect::Equal
+        if self.fetch.is_none() {
+            CardinalityEffect::Equal
+        } else {
+            CardinalityEffect::LowerEqual
+        }
     }
 
     /// Tries to swap `projection` with its input, which is known to be a
@@ -698,4 +702,21 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_cardinality_effect_with_fetch() {
+        let input = test::scan_partitioned(4);
+
+        let coalesce = CoalescePartitionsExec::new(Arc::clone(&input));
+        assert!(matches!(
+            coalesce.cardinality_effect(),
+            CardinalityEffect::Equal
+        ));
+
+        let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(10));
+        assert!(matches!(
+            coalesce.cardinality_effect(),
+            CardinalityEffect::LowerEqual
+        ));
+    }
 }
diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rs
index 04783bd9f2..42cff12903 100644
--- a/datafusion/physical-plan/src/operator_statistics/mod.rs
+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs
@@ -1986,6 +1986,22 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_passthrough_skips_coalesce_partitions_with_fetch() -> Result<()> {
+        use crate::coalesce_partitions::CoalescePartitionsExec;
+
+        let registry = StatisticsRegistry::with_providers(vec![Arc::new(
+            PassthroughStatisticsProvider,
+        )]);
+        let source = make_source(1000);
+        let coalesce: Arc<dyn ExecutionPlan> =
+            Arc::new(CoalescePartitionsExec::new(source).with_fetch(Some(10)));
+
+        let stats = compute(&registry, coalesce.as_ref())?;
+        assert_eq!(stats.base.num_rows, Precision::Exact(10));
+        Ok(())
+    }
+
     #[test]
     fn test_chain_priority() -> Result<()> {
         let mut registry = StatisticsRegistry::new();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org