You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/29 09:17:23 UTC
(datafusion) branch gh-readonly-queue/main/pr-23670-68d587468073c1fc84ffe00db66f35d506dcc828 created (now d2f7180a4a)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23670-68d587468073c1fc84ffe00db66f35d506dcc828
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at d2f7180a4a fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
This branch includes the following new commits:
new d2f7180a4a fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23670-68d587468073c1fc84ffe00db66f35d506dcc828
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit d2f7180a4a58def3b1c2da335152d4d66efad7a8
Author: Tobias Schwarzinger <to...@tuwien.ac.at>
AuthorDate: Wed Jul 29 11:16:59 2026 +0200
fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
## Which issue does this PR close?
Minor fix I stumbled across. There is no issue (I can create one if this
warrants discussion).
## Rationale for this change
If the file scan is guaranteed to have no results (`num_rows ==
Precision::Exact(0)`), filtering them does not change the result. It
will be zero anyway.
## What changes are included in this PR?
Only change `Exact` -> `Inexact` if there are non-zero rows
## Are these changes tested?
Yes.
## Are there any user-facing changes?
It can happen that the optimizer can now better optimize the query plans
(which we are looking forward to).
---
datafusion/datasource/src/file_scan_config/mod.rs | 45 ++++++++++++++++++++++-
1 file changed, 43 insertions(+), 2 deletions(-)
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 962df06302..d1dd3c11fc 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -39,6 +39,7 @@ use datafusion_execution::{
use datafusion_expr::Operator;
use crate::source::OpenArgs;
+use datafusion_common::stats::Precision;
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -1233,7 +1234,9 @@ impl FileScanConfig {
/// we can't guarantee the statistics are exact because we don't know how many
/// rows will be filtered out.
pub fn statistics(&self) -> Statistics {
- if self.file_source.filter().is_some() {
+ let filter_may_change_row_count = self.file_source.filter().is_some()
+ && self.statistics.num_rows != Precision::Exact(0);
+ if filter_may_change_row_count {
self.statistics.clone().to_inexact()
} else {
self.statistics.clone()
@@ -2415,7 +2418,6 @@ mod tests {
use crate::source::DataSourceExec;
use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
- // Create a schema with 4 columns
let schema = Arc::new(Schema::new(vec![
Field::new("col0", DataType::Int32, false),
Field::new("col1", DataType::Int32, false),
@@ -2499,6 +2501,45 @@ mod tests {
assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));
}
+ #[test]
+ fn test_statistics_with_filter() {
+ assert_num_rows_with_filter(Precision::Absent, Precision::Absent);
+ assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100));
+ assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100));
+ assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0));
+
+ /// Creates a [`FileScanConfig`] with a filter and calls [`FileScanConfig::statistics`].
+ /// Then the function checks the output num_rows stats, given the input num_rows stats.
+ fn assert_num_rows_with_filter(
+ input_num_rows: Precision<usize>,
+ expected_num_rows: Precision<usize>,
+ ) {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "col0",
+ DataType::Int32,
+ false,
+ )]));
+
+ let stats =
+ Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows);
+ let file_group =
+ FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]);
+
+ let table_schema = TableSchema::from(&schema);
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new(
+ Literal::new(ScalarValue::Boolean(Some(true))),
+ ))),
+ )
+ .with_file_groups(vec![file_group])
+ .with_statistics(stats)
+ .build();
+
+ assert_eq!(config.statistics().num_rows, expected_num_rows,);
+ }
+ }
+
/// Regression test for reusing a `DataSourceExec` after its execution-local
/// shared work queue has been drained.
///
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org