You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/04 20:30:30 UTC
(datafusion) 01/01: fix(physical-plan): set column byte_size to 0 in FilterExec zero-row interval stats (#21999)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-21999-ae796abe4e4cf6468166a539931fb0585ed06bee
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7d107f09ba0e980e8885b60187a9621726c941d4
Author: Burak Şen <bu...@gmail.com>
AuthorDate: Mon May 4 13:30:08 2026 -0700
fix(physical-plan): set column byte_size to 0 in FilterExec zero-row interval stats (#21999)
## Which issue does this PR close?
- Closes #.
## Rationale for this change
While reading code around column statistics I've realized that we send
column byte_size when there are no rows. In this case I think it is
better to set it to 0
## What changes are included in this PR?
Set `byte_size` to zero when there are no rows in filter column
statistics
## Are these changes tested?
Yes existing unit tests
## Are there any user-facing changes?
no api changes and minimal propagated statistics change
---
.../core/tests/physical_optimizer/partition_statistics.rs | 11 ++++-------
datafusion/physical-plan/src/filter.rs | 6 +++---
2 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
index f84d79146b..d06e506abf 100644
--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
@@ -381,8 +381,6 @@ mod test {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, scan)?);
let full_statistics = filter.partition_statistics(None)?;
- // Filter preserves original total_rows and byte_size from input
- // (4 total rows = 2 partitions * 2 rows each, byte_size = 4 * 4 = 16 bytes for int32)
let expected_full_statistic = Statistics {
num_rows: Precision::Inexact(0),
total_byte_size: Precision::Inexact(0),
@@ -393,7 +391,7 @@ mod test {
min_value: Precision::Exact(ScalarValue::Int32(None)),
sum_value: Precision::Exact(ScalarValue::Int32(None)),
distinct_count: Precision::Exact(0),
- byte_size: Precision::Exact(16),
+ byte_size: Precision::Exact(0),
},
ColumnStatistics {
null_count: Precision::Exact(0),
@@ -401,7 +399,7 @@ mod test {
min_value: Precision::Exact(ScalarValue::Date32(None)),
sum_value: Precision::Exact(ScalarValue::Date32(None)),
distinct_count: Precision::Exact(0),
- byte_size: Precision::Exact(16), // 4 rows * 4 bytes (Date32)
+ byte_size: Precision::Exact(0),
},
],
};
@@ -411,7 +409,6 @@ mod test {
.map(|idx| filter.partition_statistics(Some(idx)))
.collect::<Result<Vec<_>>>()?;
assert_eq!(statistics.len(), 2);
- // Per-partition stats: each partition has 2 rows, byte_size = 2 * 4 = 8
let expected_partition_statistic = Statistics {
num_rows: Precision::Inexact(0),
total_byte_size: Precision::Inexact(0),
@@ -422,7 +419,7 @@ mod test {
min_value: Precision::Exact(ScalarValue::Int32(None)),
sum_value: Precision::Exact(ScalarValue::Int32(None)),
distinct_count: Precision::Exact(0),
- byte_size: Precision::Exact(8),
+ byte_size: Precision::Exact(0),
},
ColumnStatistics {
null_count: Precision::Exact(0),
@@ -430,7 +427,7 @@ mod test {
min_value: Precision::Exact(ScalarValue::Date32(None)),
sum_value: Precision::Exact(ScalarValue::Date32(None)),
distinct_count: Precision::Exact(0),
- byte_size: Precision::Exact(8), // 2 rows * 4 bytes (Date32)
+ byte_size: Precision::Exact(0),
},
],
};
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 50efe8f509..1119d1b240 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -903,7 +903,7 @@ fn collect_new_statistics(
min_value: Precision::Exact(typed_null.clone()),
sum_value: Precision::Exact(typed_null),
distinct_count: Precision::Exact(0),
- byte_size: input_column_stats[idx].byte_size,
+ byte_size: Precision::Exact(0),
};
};
let (lower, upper) = interval.into_bounds();
@@ -1621,7 +1621,7 @@ mod tests {
sum_value: Precision::Exact(ScalarValue::Int32(None)),
distinct_count: Precision::Exact(0),
null_count: Precision::Exact(0),
- byte_size: Precision::Absent,
+ byte_size: Precision::Exact(0),
},
ColumnStatistics {
min_value: Precision::Exact(ScalarValue::Int32(None)),
@@ -1629,7 +1629,7 @@ mod tests {
sum_value: Precision::Exact(ScalarValue::Int32(None)),
distinct_count: Precision::Exact(0),
null_count: Precision::Exact(0),
- byte_size: Precision::Absent,
+ byte_size: Precision::Exact(0),
},
]
);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org