(datafusion) branch main updated: refactor(proto): destructure Arrow and Avro source serde hooks (#24696)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 7776266162 refactor(proto): destructure Arrow and Avro source serde hooks (#24696)
7776266162 is described below

commit 777626616200ec705631d1ce0ee866bffb26f28f
Author: Burak Şen <bu...@gmail.com>
AuthorDate: Sat Aug 29 09:34:29 2026 +0000

    refactor(proto): destructure Arrow and Avro source serde hooks (#24696)
    
    ## Which issue does this PR close?
    
    - Closes #24622.
    
    ## Rationale for this change
    
    Proto hooks that access fields individually can silently omit newly
    added state. Exhaustive destructuring makes such omissions compile
    errors.
    
    ## What changes are included in this PR?
    
    - Exhaustively destructure `ArrowSource` and `AvroSource` in their
    encoders.
    - Exhaustively destructure their protobuf nodes in the decoders.
    - Document why non-serialized fields are excluded.
    
    The wire format is unchanged.
    
    ## Are these changes tested?
    
    Covered by the existing Arrow and Avro round-trip tests:
    
    ```bash
    cargo test -p datafusion-proto --test proto_integration --features avro roundtrip_arrow
    cargo test -p datafusion-proto --test proto_integration --features avro roundtrip_avro
    ```
    
    The standard lint suite also passes:
    
    ```bash
    ./dev/rust_lint.sh
    ```
    
    ## Are there any user-facing changes?
    
    No.
---
 datafusion/datasource-arrow/src/source.rs | 23 +++++++++++++++++++----
 datafusion/datasource-avro/src/source.rs  | 18 +++++++++++++++++-
 2 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs
index b909263748..7e8189a78d 100644
--- a/datafusion/datasource-arrow/src/source.rs
+++ b/datafusion/datasource-arrow/src/source.rs
@@ -414,7 +414,20 @@ impl FileSource for ArrowSource {
         use datafusion_proto_models::protobuf;
         use protobuf::physical_plan_node::PhysicalPlanType;
 
-        let format = match self.format {
+        // Exhaustive destructure: adding a field to `ArrowSource` without
+        // deciding how it is serialized is a compile error, not a silent
+        // round-trip gap.
+        let Self {
+            format,
+            // Runtime metrics, not part of the plan.
+            metrics: _,
+            // Serialized in `base` and reapplied on decode.
+            projection: _,
+            // Serialized in `base` and used to rebuild the source on decode.
+            table_schema: _,
+        } = self;
+
+        let format = match format {
             ArrowFormat::File => protobuf::ArrowIpcFormat::File,
             ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream,
         };
@@ -452,16 +465,18 @@ impl ArrowSource {
             );
         };
 
-        let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
+        let protobuf::ArrowScanExecNode { base_conf, format } = scan;
+
+        let base_conf = base_conf.as_ref().ok_or_else(|| {
             datafusion_common::internal_datafusion_err!(
                 "ArrowScanExecNode is missing required field 'base_conf'"
             )
         })?;
 
-        let format = protobuf::ArrowIpcFormat::try_from(scan.format).map_err(|_| {
+        let format = protobuf::ArrowIpcFormat::try_from(*format).map_err(|_| {
             datafusion_common::internal_datafusion_err!(
                 "Unknown ArrowIpcFormat: {}",
-                scan.format
+                format
             )
         })?;
 
diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs
index 7c29adad2b..e42a97e80c 100644
--- a/datafusion/datasource-avro/src/source.rs
+++ b/datafusion/datasource-avro/src/source.rs
@@ -189,6 +189,20 @@ impl FileSource for AvroSource {
         use datafusion_proto_models::protobuf;
         use protobuf::physical_plan_node::PhysicalPlanType;
 
+        // Exhaustive destructure: adding a field to `AvroSource` without
+        // deciding how it is serialized is a compile error, not a silent
+        // round-trip gap.
+        let Self {
+            // Serialized in `base` and used to rebuild the source on decode.
+            table_schema: _,
+            // Set from `FileScanConfig` when the scan is opened.
+            batch_size: _,
+            // Serialized in `base` and reapplied on decode.
+            projection: _,
+            // Runtime metrics, not part of the plan.
+            metrics: _,
+        } = self;
+
         let node = protobuf::AvroScanExecNode {
             base_conf: Some(base.try_to_proto(ctx)?),
         };
@@ -216,7 +230,9 @@ impl AvroSource {
             );
         };
 
-        let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
+        let protobuf::AvroScanExecNode { base_conf } = scan;
+
+        let base_conf = base_conf.as_ref().ok_or_else(|| {
             datafusion_common::internal_datafusion_err!(
                 "AvroScanExecNode is missing required field 'base_conf'"
             )


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org