You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/30 16:51:40 UTC
(datafusion) branch gh-readonly-queue/main/pr-23999-541caab85a265c104888f1988fe54b08fa022f53 created (now 833e501459)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23999-541caab85a265c104888f1988fe54b08fa022f53
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 833e501459 fix(proto): prevent duplicate partition statistics on roundtrip (#23999)
This branch includes the following new commits:
new 833e501459 fix(proto): prevent duplicate partition statistics on roundtrip (#23999)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix(proto): prevent duplicate partition statistics on roundtrip (#23999)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23999-541caab85a265c104888f1988fe54b08fa022f53
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 833e501459dcc418042b3280a0ea9cb2980005d8
Author: Burak Şen <bu...@gmail.com>
AuthorDate: Thu Jul 30 19:38:35 2026 +0300
fix(proto): prevent duplicate partition statistics on roundtrip (#23999)
## Which issue does this PR close?
- Closes #23998
## Rationale for this change
Decoding a `PartitionedFile` re-appended partition-column statistics
already present in protobuf.
## What changes are included in this PR?
Assign decoded statistics directly.
## Are these changes tested?
Yes new roundtrip test for this
## Are there any user-facing changes?
no
---------
Co-authored-by: Adrian Garcia Badaracco <17...@users.noreply.github.com>
---
datafusion/proto/src/physical_plan/from_proto.rs | 30 +++++++++++++++++++++---
1 file changed, 27 insertions(+), 3 deletions(-)
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index d5a1e0efac..7aa6376313 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -660,7 +660,10 @@ impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile {
pf = pf.with_range(file_range.start, file_range.end);
}
if let Some(proto_stats) = val.statistics.as_ref() {
- pf = pf.with_statistics(Arc::new(proto_stats.try_into()?));
+ // The wire format carries statistics for the full table schema (file + partition
+ // columns), so assign directly — `with_statistics` would append the partition
+ // column stats a second time.
+ pf.statistics = Some(Arc::new(proto_stats.try_into()?));
}
Ok(pf)
}
@@ -807,8 +810,8 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
#[cfg(test)]
mod tests {
-
use super::*;
+ use arrow::datatypes::{DataType, Field, Schema};
#[test]
fn partitioned_file_path_roundtrip_percent_encoded() {
@@ -833,7 +836,6 @@ mod tests {
#[test]
fn partitioned_file_arrow_schema_roundtrip() {
- use arrow::datatypes::{DataType, Field, Schema};
use std::collections::HashMap;
let arrow_schema = Arc::new(Schema::new_with_metadata(
@@ -858,6 +860,28 @@ mod tests {
);
}
+ #[test]
+ fn partitioned_file_statistics_roundtrip_with_partition_values() {
+ use datafusion_common::Statistics;
+ let file_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
+ let pf = PartitionedFile::new("foo/bar.parquet", 1234)
+ .with_partition_values(vec![ScalarValue::from("2024-01-01")])
+ .with_statistics(Arc::new(Statistics::new_unknown(&file_schema)));
+
+ // `statistics` covers the full table schema: file columns followed by one
+ // entry per partition column.
+ let expected_len = file_schema.fields().len() + pf.partition_values.len();
+ assert_eq!(
+ pf.statistics.as_ref().unwrap().column_statistics.len(),
+ expected_len
+ );
+
+ let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
+ let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
+
+ assert_eq!(decoded.statistics, pf.statistics);
+ }
+
#[test]
fn partitioned_file_from_proto_invalid_path() {
let proto = protobuf::PartitionedFile {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org