(datafusion) 01/01: fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23670-68d587468073c1fc84ffe00db66f35d506dcc828
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit d2f7180a4a58def3b1c2da335152d4d66efad7a8
Author: Tobias Schwarzinger <to...@tuwien.ac.at>
AuthorDate: Wed Jul 29 11:16:59 2026 +0200

    fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
    
    ## Which issue does this PR close?
    
    Minor fix I stumbled across. There is no issue (I can create one if this
    warrants discussion).
    
    ## Rationale for this change
    
    If the file scan is guaranteed to have no results (`num_rows ==
    Precision::Exact(0)`), filtering them does not change the result. It
    will be zero anyway.
    
    ## What changes are included in this PR?
    
    Only change `Exact` -> `Inexact` if there are non-zero rows
    
    ## Are these changes tested?
    
    Yes.
    
    ## Are there any user-facing changes?
    
    It can happen that the optimizer can now better optimize the query plans
    (which we are looking forward to).
---
 datafusion/datasource/src/file_scan_config/mod.rs | 45 ++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 962df06302..d1dd3c11fc 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -39,6 +39,7 @@ use datafusion_execution::{
 use datafusion_expr::Operator;
 
 use crate::source::OpenArgs;
+use datafusion_common::stats::Precision;
 use datafusion_physical_expr::expressions::{BinaryExpr, Column};
 use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
 use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -1233,7 +1234,9 @@ impl FileScanConfig {
     /// we can't guarantee the statistics are exact because we don't know how many
     /// rows will be filtered out.
     pub fn statistics(&self) -> Statistics {
-        if self.file_source.filter().is_some() {
+        let filter_may_change_row_count = self.file_source.filter().is_some()
+            && self.statistics.num_rows != Precision::Exact(0);
+        if filter_may_change_row_count {
             self.statistics.clone().to_inexact()
         } else {
             self.statistics.clone()
@@ -2415,7 +2418,6 @@ mod tests {
         use crate::source::DataSourceExec;
         use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
 
-        // Create a schema with 4 columns
         let schema = Arc::new(Schema::new(vec![
             Field::new("col0", DataType::Int32, false),
             Field::new("col1", DataType::Int32, false),
@@ -2499,6 +2501,45 @@ mod tests {
         assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));
     }
 
+    #[test]
+    fn test_statistics_with_filter() {
+        assert_num_rows_with_filter(Precision::Absent, Precision::Absent);
+        assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100));
+        assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100));
+        assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0));
+
+        /// Creates a [`FileScanConfig`] with a filter and calls [`FileScanConfig::statistics`].
+        /// Then the function checks the output num_rows stats, given the input num_rows stats.
+        fn assert_num_rows_with_filter(
+            input_num_rows: Precision<usize>,
+            expected_num_rows: Precision<usize>,
+        ) {
+            let schema = Arc::new(Schema::new(vec![Field::new(
+                "col0",
+                DataType::Int32,
+                false,
+            )]));
+
+            let stats =
+                Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows);
+            let file_group =
+                FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]);
+
+            let table_schema = TableSchema::from(&schema);
+            let config = FileScanConfigBuilder::new(
+                ObjectStoreUrl::parse("test:///").unwrap(),
+                Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new(
+                    Literal::new(ScalarValue::Boolean(Some(true))),
+                ))),
+            )
+            .with_file_groups(vec![file_group])
+            .with_statistics(stats)
+            .build();
+
+            assert_eq!(config.statistics().num_rows, expected_num_rows,);
+        }
+    }
+
     /// Regression test for reusing a `DataSourceExec` after its execution-local
     /// shared work queue has been drained.
     ///


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org