You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/22 02:13:37 UTC

(datafusion) branch main updated: chore: remove get_statistics_with_limit (#24549)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 7538aefbe9 chore: remove get_statistics_with_limit (#24549)
7538aefbe9 is described below

commit 7538aefbe9ee224814570798cd381d1948029efa
Author: Yongting You <20...@gmail.com>
AuthorDate: Sat Aug 22 01:57:26 2026 +0000

    chore: remove get_statistics_with_limit (#24549)
    
    ## Which issue does this PR close?
    
    Part of #24542.
    
    ## Rationale for this change
    
    This API is past DataFusion's [deprecation
    period](https://datafusion.apache.org/contributor-guide/api-health.html#deprecation-guidelines).
    
    ## What changes are included in this PR?
    
    Removes `get_statistics_with_limit` and its supporting code. Use
    `get_files_with_limit` and `compute_all_files_statistics` instead.
    
    ## Are these changes tested?
    
    NA
    ## Are there any user-facing changes?
    
    Yes, this is a breaking Rust API change.
---
 datafusion/datasource/src/statistics.rs | 417 +-------------------------------
 1 file changed, 4 insertions(+), 413 deletions(-)

diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs
index 4218823026..48d180364f 100644
--- a/datafusion/datasource/src/statistics.rs
+++ b/datafusion/datasource/src/statistics.rs
@@ -29,15 +29,13 @@ use arrow::array::RecordBatch;
 use arrow::compute::SortColumn;
 use arrow::datatypes::SchemaRef;
 use arrow::row::{Row, Rows};
-use datafusion_common::stats::{NdvFallback, Precision};
+use datafusion_common::stats::NdvFallback;
 use datafusion_common::{
     DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,
 };
 use datafusion_physical_expr::expressions::Column;
 use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
-use datafusion_physical_plan::{ColumnStatistics, Statistics};
-
-use futures::{Stream, StreamExt};
+use datafusion_physical_plan::Statistics;
 
 /// A normalized representation of file min/max statistics that allows for efficient sorting & comparison.
 /// The min/max values are ordered by [`Self::sort_order`].
@@ -290,172 +288,6 @@ fn sort_columns_from_physical_sort_exprs(
         .collect()
 }
 
-fn seed_summary_statistics(summary_statistics: &mut Statistics, file_stats: &Statistics) {
-    summary_statistics.num_rows = file_stats.num_rows;
-    summary_statistics.total_byte_size = file_stats.total_byte_size;
-
-    for (summary_col_stats, file_col_stats) in summary_statistics
-        .column_statistics
-        .iter_mut()
-        .zip(file_stats.column_statistics.iter())
-    {
-        summary_col_stats.null_count = file_col_stats.null_count;
-        summary_col_stats.max_value = file_col_stats.max_value.clone();
-        summary_col_stats.min_value = file_col_stats.min_value.clone();
-        summary_col_stats.sum_value = file_col_stats.sum_value.cast_to_sum_type();
-        summary_col_stats.byte_size = file_col_stats.byte_size;
-    }
-}
-
-fn merge_summary_statistics(
-    summary_statistics: &mut Statistics,
-    file_stats: &Statistics,
-) {
-    summary_statistics.num_rows = summary_statistics.num_rows.add(&file_stats.num_rows);
-    summary_statistics.total_byte_size = summary_statistics
-        .total_byte_size
-        .add(&file_stats.total_byte_size);
-
-    for (summary_col_stats, file_col_stats) in summary_statistics
-        .column_statistics
-        .iter_mut()
-        .zip(file_stats.column_statistics.iter())
-    {
-        let ColumnStatistics {
-            null_count: file_nc,
-            max_value: file_max,
-            min_value: file_min,
-            sum_value: file_sum,
-            distinct_count: _,
-            byte_size: file_sbs,
-        } = file_col_stats;
-
-        summary_col_stats.null_count = summary_col_stats.null_count.add(file_nc);
-        summary_col_stats.max_value = summary_col_stats.max_value.max(file_max);
-        summary_col_stats.min_value = summary_col_stats.min_value.min(file_min);
-        summary_col_stats.sum_value = summary_col_stats.sum_value.add_for_sum(file_sum);
-        summary_col_stats.byte_size = summary_col_stats.byte_size.add(file_sbs);
-    }
-}
-
-fn seed_first_file_statistics(
-    limit_num_rows: &mut Precision<usize>,
-    summary_statistics: &mut Statistics,
-    file_stats: &Statistics,
-    collect_stats: bool,
-) {
-    *limit_num_rows = file_stats.num_rows;
-
-    if collect_stats {
-        seed_summary_statistics(summary_statistics, file_stats);
-    }
-}
-
-fn merge_file_statistics(
-    limit_num_rows: &mut Precision<usize>,
-    summary_statistics: &mut Statistics,
-    file_stats: &Statistics,
-    collect_stats: bool,
-) {
-    *limit_num_rows = limit_num_rows.add(&file_stats.num_rows);
-
-    if collect_stats {
-        merge_summary_statistics(summary_statistics, file_stats);
-    }
-}
-
-/// Get all files as well as the file level summary statistics (no statistic for partition columns).
-/// If the optional `limit` is provided, includes only sufficient files. Needed to read up to
-/// `limit` number of rows. `collect_stats` is passed down from the configuration parameter on
-/// `ListingTable`. If it is false we only construct bare statistics and skip a potentially expensive
-///  call to `multiunzip` for constructing file level summary statistics.
-#[deprecated(
-    since = "47.0.0",
-    note = "Please use `get_files_with_limit` and  `compute_all_files_statistics` instead"
-)]
-#[cfg_attr(not(test), expect(unused))]
-pub async fn get_statistics_with_limit(
-    all_files: impl Stream<Item = Result<(PartitionedFile, Arc<Statistics>)>>,
-    file_schema: SchemaRef,
-    limit: Option<usize>,
-    collect_stats: bool,
-) -> Result<(FileGroup, Statistics)> {
-    let mut result_files = FileGroup::default();
-    // These statistics can be calculated as long as at least one file provides
-    // useful information. If none of the files provides any information, then
-    // they will end up having `Precision::Absent` values. Throughout calculations,
-    // missing values will be imputed as:
-    // - zero for summations, and
-    // - neutral element for extreme points.
-    let size = file_schema.fields().len();
-    let mut summary_statistics = Statistics {
-        num_rows: Precision::Absent,
-        total_byte_size: Precision::Absent,
-        column_statistics: vec![ColumnStatistics::default(); size],
-    };
-    // Keep limit pruning separate from the returned summary so `collect_stats=false`
-    // can still stop early using known file row counts.
-    let mut limit_num_rows = Precision::<usize>::Absent;
-
-    // Fusing the stream allows us to call next safely even once it is finished.
-    let mut all_files = Box::pin(all_files.fuse());
-
-    if let Some(first_file) = all_files.next().await {
-        let (mut file, file_stats) = first_file?;
-        file.statistics = Some(Arc::clone(&file_stats));
-        result_files.push(file);
-
-        seed_first_file_statistics(
-            &mut limit_num_rows,
-            &mut summary_statistics,
-            &file_stats,
-            collect_stats,
-        );
-
-        // If the number of rows exceeds the limit, we can stop processing
-        // files. This only applies when we know the number of rows. It also
-        // currently ignores tables that have no statistics regarding the
-        // number of rows.
-        let conservative_num_rows = match limit_num_rows {
-            Precision::Exact(nr) => nr,
-            _ => usize::MIN,
-        };
-        if conservative_num_rows <= limit.unwrap_or(usize::MAX) {
-            while let Some(current) = all_files.next().await {
-                let (mut file, file_stats) = current?;
-                file.statistics = Some(Arc::clone(&file_stats));
-                result_files.push(file);
-                merge_file_statistics(
-                    &mut limit_num_rows,
-                    &mut summary_statistics,
-                    &file_stats,
-                    collect_stats,
-                );
-
-                // If the number of rows exceeds the limit, we can stop processing
-                // files. This only applies when we know the number of rows. It also
-                // currently ignores tables that have no statistics regarding the
-                // number of rows.
-                if limit_num_rows.get_value().unwrap_or(&usize::MIN)
-                    > &limit.unwrap_or(usize::MAX)
-                {
-                    break;
-                }
-            }
-        }
-    };
-
-    let mut statistics = summary_statistics;
-    if all_files.next().await.is_some() {
-        // If we still have files in the stream, it means that the limit kicked
-        // in, and the statistic could have been different had we processed the
-        // files in a different order.
-        statistics = statistics.to_inexact()
-    }
-
-    Ok((result_files, statistics))
-}
-
 /// Computes the summary statistics for a group of files(`FileGroup` level's statistics).
 ///
 /// This function combines statistics from all files in the file group to create
@@ -553,56 +385,13 @@ mod tests {
     use crate::PartitionedFile;
     use crate::file_groups::FileGroup;
     use arrow::datatypes::{DataType, Field, Schema};
-    use futures::stream;
-
-    fn file_stats(sum: u32) -> Statistics {
-        Statistics {
-            num_rows: Precision::Exact(1),
-            total_byte_size: Precision::Exact(4),
-            column_statistics: vec![ColumnStatistics {
-                null_count: Precision::Exact(0),
-                max_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),
-                min_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),
-                sum_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),
-                distinct_count: Precision::Exact(1),
-                byte_size: Precision::Exact(4),
-            }],
-        }
-    }
+    use datafusion_common::stats::Precision;
+    use datafusion_physical_plan::ColumnStatistics;
 
     fn test_schema() -> SchemaRef {
         Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, true)]))
     }
 
-    fn make_file_stats(
-        num_rows: usize,
-        total_byte_size: usize,
-        col_stats: ColumnStatistics,
-    ) -> Arc<Statistics> {
-        Arc::new(Statistics {
-            num_rows: Precision::Exact(num_rows),
-            total_byte_size: Precision::Exact(total_byte_size),
-            column_statistics: vec![col_stats],
-        })
-    }
-
-    fn rich_col_stats(
-        null_count: usize,
-        min: i64,
-        max: i64,
-        sum: i64,
-        byte_size: usize,
-    ) -> ColumnStatistics {
-        ColumnStatistics {
-            null_count: Precision::Exact(null_count),
-            max_value: Precision::Exact(ScalarValue::Int64(Some(max))),
-            min_value: Precision::Exact(ScalarValue::Int64(Some(min))),
-            distinct_count: Precision::Absent,
-            sum_value: Precision::Exact(ScalarValue::Int64(Some(sum))),
-            byte_size: Precision::Exact(byte_size),
-        }
-    }
-
     fn utf8_file_stats(ndv: usize, min: &str, max: &str) -> Statistics {
         Statistics {
             num_rows: Precision::Exact(1),
@@ -621,204 +410,6 @@ mod tests {
     fn file_with_stats(path: &str, stats: Statistics) -> PartitionedFile {
         PartitionedFile::new(path, 1).with_statistics(Arc::new(stats))
     }
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn test_get_statistics_with_limit_casts_first_file_sum_to_sum_type()
-    -> Result<()> {
-        let schema =
-            Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));
-
-        let files = stream::iter(vec![Ok((
-            PartitionedFile::new("f1.parquet", 1),
-            Arc::new(file_stats(100)),
-        ))]);
-
-        let (_group, stats) =
-            get_statistics_with_limit(files, schema, None, true).await?;
-
-        assert_eq!(
-            stats.column_statistics[0].sum_value,
-            Precision::Exact(ScalarValue::UInt64(Some(100)))
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn test_get_statistics_with_limit_merges_sum_with_unsigned_widening()
-    -> Result<()> {
-        let schema =
-            Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));
-
-        let files = stream::iter(vec![
-            Ok((
-                PartitionedFile::new("f1.parquet", 1),
-                Arc::new(file_stats(100)),
-            )),
-            Ok((
-                PartitionedFile::new("f2.parquet", 1),
-                Arc::new(file_stats(200)),
-            )),
-        ]);
-
-        let (_group, stats) =
-            get_statistics_with_limit(files, schema, None, true).await?;
-
-        assert_eq!(
-            stats.column_statistics[0].sum_value,
-            Precision::Exact(ScalarValue::UInt64(Some(300)))
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn get_statistics_with_limit_collect_stats_false_returns_bare_statistics() {
-        let all_files = stream::iter(vec![
-            Ok((
-                PartitionedFile::new("first.parquet", 10),
-                make_file_stats(0, 0, rich_col_stats(1, 1, 9, 15, 64)),
-            )),
-            Ok((
-                PartitionedFile::new("second.parquet", 20),
-                make_file_stats(10, 100, rich_col_stats(2, 10, 99, 300, 128)),
-            )),
-        ]);
-
-        let (_files, statistics) =
-            get_statistics_with_limit(all_files, test_schema(), None, false)
-                .await
-                .unwrap();
-
-        assert_eq!(statistics.num_rows, Precision::Absent);
-        assert_eq!(statistics.total_byte_size, Precision::Absent);
-        assert_eq!(statistics.column_statistics.len(), 1);
-        assert_eq!(
-            statistics.column_statistics[0].null_count,
-            Precision::Absent
-        );
-        assert_eq!(statistics.column_statistics[0].max_value, Precision::Absent);
-        assert_eq!(statistics.column_statistics[0].min_value, Precision::Absent);
-        assert_eq!(statistics.column_statistics[0].sum_value, Precision::Absent);
-        assert_eq!(statistics.column_statistics[0].byte_size, Precision::Absent);
-    }
-
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn get_statistics_with_limit_collect_stats_false_uses_row_counts_for_limit() {
-        let all_files = stream::iter(vec![
-            Ok((
-                PartitionedFile::new("first.parquet", 10),
-                make_file_stats(3, 30, rich_col_stats(1, 1, 9, 15, 64)),
-            )),
-            Ok((
-                PartitionedFile::new("second.parquet", 20),
-                make_file_stats(3, 30, rich_col_stats(2, 10, 99, 300, 128)),
-            )),
-            Ok((
-                PartitionedFile::new("third.parquet", 30),
-                make_file_stats(3, 30, rich_col_stats(0, 100, 199, 450, 256)),
-            )),
-        ]);
-
-        let (files, statistics) =
-            get_statistics_with_limit(all_files, test_schema(), Some(4), false)
-                .await
-                .unwrap();
-
-        assert_eq!(files.len(), 2);
-        assert_eq!(statistics.num_rows, Precision::Absent);
-        assert_eq!(statistics.total_byte_size, Precision::Absent);
-    }
-
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn get_statistics_with_limit_collect_stats_true_aggregates_statistics() {
-        let all_files = stream::iter(vec![
-            Ok((
-                PartitionedFile::new("first.parquet", 10),
-                make_file_stats(5, 50, rich_col_stats(1, 1, 9, 15, 64)),
-            )),
-            Ok((
-                PartitionedFile::new("second.parquet", 20),
-                make_file_stats(10, 100, rich_col_stats(2, 10, 99, 300, 128)),
-            )),
-        ]);
-
-        let (_files, statistics) =
-            get_statistics_with_limit(all_files, test_schema(), None, true)
-                .await
-                .unwrap();
-
-        assert_eq!(statistics.num_rows, Precision::Exact(15));
-        assert_eq!(statistics.total_byte_size, Precision::Exact(150));
-        assert_eq!(
-            statistics.column_statistics[0].null_count,
-            Precision::Exact(3)
-        );
-        assert_eq!(
-            statistics.column_statistics[0].min_value,
-            Precision::Exact(ScalarValue::Int64(Some(1)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].max_value,
-            Precision::Exact(ScalarValue::Int64(Some(99)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].sum_value,
-            Precision::Exact(ScalarValue::Int64(Some(315)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].byte_size,
-            Precision::Exact(192)
-        );
-    }
-
-    #[tokio::test]
-    #[expect(deprecated)]
-    async fn get_statistics_with_limit_collect_stats_true_limit_marks_inexact() {
-        let all_files = stream::iter(vec![
-            Ok((
-                PartitionedFile::new("first.parquet", 10),
-                make_file_stats(5, 50, rich_col_stats(0, 1, 5, 15, 64)),
-            )),
-            Ok((
-                PartitionedFile::new("second.parquet", 20),
-                make_file_stats(5, 50, rich_col_stats(1, 6, 10, 40, 64)),
-            )),
-            Ok((
-                PartitionedFile::new("third.parquet", 20),
-                make_file_stats(5, 50, rich_col_stats(2, 11, 15, 65, 64)),
-            )),
-        ]);
-
-        let (files, statistics) =
-            get_statistics_with_limit(all_files, test_schema(), Some(8), true)
-                .await
-                .unwrap();
-
-        assert_eq!(files.len(), 2);
-        assert_eq!(statistics.num_rows, Precision::Inexact(10));
-        assert_eq!(statistics.total_byte_size, Precision::Inexact(100));
-        assert_eq!(
-            statistics.column_statistics[0].min_value,
-            Precision::Inexact(ScalarValue::Int64(Some(1)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].max_value,
-            Precision::Inexact(ScalarValue::Int64(Some(10)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].sum_value,
-            Precision::Inexact(ScalarValue::Int64(Some(55)))
-        );
-        assert_eq!(
-            statistics.column_statistics[0].byte_size,
-            Precision::Inexact(128)
-        );
-    }
 
     #[test]
     fn test_compute_file_group_statistics_uses_max_ndv_fallback() -> Result<()> {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org