You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by xu...@apache.org on 2025/03/27 02:36:50 UTC

(datafusion) branch main updated: Add `downcast_to_source` method for `DataSourceExec` (#15416)

This is an automated email from the ASF dual-hosted git repository.

xudong963 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new fdb4e848b6 Add `downcast_to_source` method for `DataSourceExec` (#15416)
fdb4e848b6 is described below

commit fdb4e848b65c001dd3f65b477296e07cbe8e0b07
Author: xudong.w <wx...@gmail.com>
AuthorDate: Thu Mar 27 10:36:46 2025 +0800

    Add `downcast_to_source` method for `DataSourceExec` (#15416)
    
    * Add downcast_to_source method for DataSourceExec
    
    * rename
    
    * fix conflicts
    
    * fix cippy
    
    * add the change to upgrading doc
    
    * prettier
    
    * remove
    
    * address comments
---
 .../examples/parquet_exec_visitor.rs               |  26 ++--
 datafusion/core/src/test_util/parquet.rs           |  15 +--
 datafusion/core/tests/parquet/utils.rs             |  17 +--
 datafusion/core/tests/sql/path_partition.rs        |  41 +++---
 datafusion/datasource/src/source.rs                |  19 +++
 datafusion/proto/src/physical_plan/mod.rs          |  44 +++----
 datafusion/substrait/src/physical_plan/producer.rs | 141 ++++++++++-----------
 7 files changed, 141 insertions(+), 162 deletions(-)

diff --git a/datafusion-examples/examples/parquet_exec_visitor.rs b/datafusion-examples/examples/parquet_exec_visitor.rs
index 8133457dd9..84f92d4f45 100644
--- a/datafusion-examples/examples/parquet_exec_visitor.rs
+++ b/datafusion-examples/examples/parquet_exec_visitor.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 
 use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::listing::ListingOptions;
-use datafusion::datasource::physical_plan::{FileGroup, FileScanConfig, ParquetSource};
+use datafusion::datasource::physical_plan::{FileGroup, ParquetSource};
 use datafusion::datasource::source::DataSourceExec;
 use datafusion::error::DataFusionError;
 use datafusion::execution::context::SessionContext;
@@ -98,24 +98,16 @@ impl ExecutionPlanVisitor for ParquetExecVisitor {
     fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
         // If needed match on a specific `ExecutionPlan` node type
         if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
-            let data_source = data_source_exec.data_source();
-            if let Some(file_config) =
-                data_source.as_any().downcast_ref::<FileScanConfig>()
+            if let Some((file_config, _)) =
+                data_source_exec.downcast_to_file_source::<ParquetSource>()
             {
-                if file_config
-                    .file_source()
-                    .as_any()
-                    .downcast_ref::<ParquetSource>()
-                    .is_some()
-                {
-                    self.file_groups = Some(file_config.file_groups.clone());
+                self.file_groups = Some(file_config.file_groups.clone());
 
-                    let metrics = match data_source_exec.metrics() {
-                        None => return Ok(true),
-                        Some(metrics) => metrics,
-                    };
-                    self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
-                }
+                let metrics = match data_source_exec.metrics() {
+                    None => return Ok(true),
+                    Some(metrics) => metrics,
+                };
+                self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
             }
         }
         Ok(true)
diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rs
index c0be13baf2..8c046460c2 100644
--- a/datafusion/core/src/test_util/parquet.rs
+++ b/datafusion/core/src/test_util/parquet.rs
@@ -201,18 +201,11 @@ impl TestParquetFile {
     /// on the first one it finds
     pub fn parquet_metrics(plan: &Arc<dyn ExecutionPlan>) -> Option<MetricsSet> {
         if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
-            let data_source = data_source_exec.data_source();
-            if let Some(maybe_parquet) =
-                data_source.as_any().downcast_ref::<FileScanConfig>()
+            if data_source_exec
+                .downcast_to_file_source::<ParquetSource>()
+                .is_some()
             {
-                if maybe_parquet
-                    .file_source()
-                    .as_any()
-                    .downcast_ref::<ParquetSource>()
-                    .is_some()
-                {
-                    return data_source_exec.metrics();
-                }
+                return data_source_exec.metrics();
             }
         }
 
diff --git a/datafusion/core/tests/parquet/utils.rs b/datafusion/core/tests/parquet/utils.rs
index b16d420bf7..24b6cadc14 100644
--- a/datafusion/core/tests/parquet/utils.rs
+++ b/datafusion/core/tests/parquet/utils.rs
@@ -17,7 +17,7 @@
 
 //! Utilities for parquet tests
 
-use datafusion::datasource::physical_plan::{FileScanConfig, ParquetSource};
+use datafusion::datasource::physical_plan::ParquetSource;
 use datafusion::datasource::source::DataSourceExec;
 use datafusion_physical_plan::metrics::MetricsSet;
 use datafusion_physical_plan::{accept, ExecutionPlan, ExecutionPlanVisitor};
@@ -48,18 +48,11 @@ impl ExecutionPlanVisitor for MetricsFinder {
     type Error = std::convert::Infallible;
     fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
         if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
-            let data_source = data_source_exec.data_source();
-            if let Some(file_config) =
-                data_source.as_any().downcast_ref::<FileScanConfig>()
+            if data_source_exec
+                .downcast_to_file_source::<ParquetSource>()
+                .is_some()
             {
-                if file_config
-                    .file_source()
-                    .as_any()
-                    .downcast_ref::<ParquetSource>()
-                    .is_some()
-                {
-                    self.metrics = data_source_exec.metrics();
-                }
+                self.metrics = data_source_exec.metrics();
             }
         }
         // stop searching once we have found the metrics
diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs
index 46aecd1dc0..bf8466d849 100644
--- a/datafusion/core/tests/sql/path_partition.rs
+++ b/datafusion/core/tests/sql/path_partition.rs
@@ -25,7 +25,7 @@ use std::sync::Arc;
 
 use arrow::datatypes::DataType;
 use datafusion::datasource::listing::ListingTableUrl;
-use datafusion::datasource::physical_plan::{FileScanConfig, ParquetSource};
+use datafusion::datasource::physical_plan::ParquetSource;
 use datafusion::datasource::source::DataSourceExec;
 use datafusion::{
     datasource::{
@@ -87,29 +87,22 @@ async fn parquet_partition_pruning_filter() -> Result<()> {
     ];
     let exec = table.scan(&ctx.state(), None, &filters, None).await?;
     let data_source_exec = exec.as_any().downcast_ref::<DataSourceExec>().unwrap();
-    let data_source = data_source_exec.data_source();
-    let file_source = data_source
-        .as_any()
-        .downcast_ref::<FileScanConfig>()
-        .unwrap();
-    let parquet_config = file_source
-        .file_source()
-        .as_any()
-        .downcast_ref::<ParquetSource>()
-        .unwrap();
-    let pred = parquet_config.predicate().unwrap();
-    // Only the last filter should be pushdown to TableScan
-    let expected = Arc::new(BinaryExpr::new(
-        Arc::new(Column::new_with_schema("id", &exec.schema()).unwrap()),
-        Operator::Gt,
-        Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
-    ));
-
-    assert!(pred.as_any().is::<BinaryExpr>());
-    let pred = pred.as_any().downcast_ref::<BinaryExpr>().unwrap();
-
-    assert_eq!(pred, expected.as_ref());
-
+    if let Some((_, parquet_config)) =
+        data_source_exec.downcast_to_file_source::<ParquetSource>()
+    {
+        let pred = parquet_config.predicate().unwrap();
+        // Only the last filter should be pushdown to TableScan
+        let expected = Arc::new(BinaryExpr::new(
+            Arc::new(Column::new_with_schema("id", &exec.schema()).unwrap()),
+            Operator::Gt,
+            Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
+        ));
+
+        assert!(pred.as_any().is::<BinaryExpr>());
+        let pred = pred.as_any().downcast_ref::<BinaryExpr>().unwrap();
+
+        assert_eq!(pred, expected.as_ref());
+    }
     Ok(())
 }
 
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 6e78df760d..74fd123e8d 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -29,6 +29,7 @@ use datafusion_physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
 };
 
+use crate::file_scan_config::FileScanConfig;
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::{Constraints, Statistics};
 use datafusion_execution::{SendableRecordBatchStream, TaskContext};
@@ -230,4 +231,22 @@ impl DataSourceExec {
             Boundedness::Bounded,
         )
     }
+
+    /// Downcast the `DataSourceExec`'s `data_source` to a specific file source
+    ///
+    /// Returns `None` if
+    /// 1. the datasource is not scanning files (`FileScanConfig`)
+    /// 2. The [`FileScanConfig::file_source`] is not of type `T`
+    pub fn downcast_to_file_source<T: 'static>(&self) -> Option<(&FileScanConfig, &T)> {
+        self.data_source()
+            .as_any()
+            .downcast_ref::<FileScanConfig>()
+            .and_then(|file_scan_conf| {
+                file_scan_conf
+                    .file_source()
+                    .as_any()
+                    .downcast_ref::<T>()
+                    .map(|source| (file_scan_conf, source))
+            })
+    }
 }
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 14b169629f..7712c018f3 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -1714,31 +1714,27 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
 
         #[cfg(feature = "parquet")]
         if let Some(exec) = plan.downcast_ref::<DataSourceExec>() {
-            let data_source_exec = exec.data_source();
-            if let Some(maybe_parquet) =
-                data_source_exec.as_any().downcast_ref::<FileScanConfig>()
+            if let Some((maybe_parquet, conf)) =
+                exec.downcast_to_file_source::<ParquetSource>()
             {
-                let source = maybe_parquet.file_source();
-                if let Some(conf) = source.as_any().downcast_ref::<ParquetSource>() {
-                    let predicate = conf
-                        .predicate()
-                        .map(|pred| serialize_physical_expr(pred, extension_codec))
-                        .transpose()?;
-                    return Ok(protobuf::PhysicalPlanNode {
-                        physical_plan_type: Some(PhysicalPlanType::ParquetScan(
-                            protobuf::ParquetScanExecNode {
-                                base_conf: Some(serialize_file_scan_config(
-                                    maybe_parquet,
-                                    extension_codec,
-                                )?),
-                                predicate,
-                                parquet_options: Some(
-                                    conf.table_parquet_options().try_into()?,
-                                ),
-                            },
-                        )),
-                    });
-                }
+                let predicate = conf
+                    .predicate()
+                    .map(|pred| serialize_physical_expr(pred, extension_codec))
+                    .transpose()?;
+                return Ok(protobuf::PhysicalPlanNode {
+                    physical_plan_type: Some(PhysicalPlanType::ParquetScan(
+                        protobuf::ParquetScanExecNode {
+                            base_conf: Some(serialize_file_scan_config(
+                                maybe_parquet,
+                                extension_codec,
+                            )?),
+                            predicate,
+                            parquet_options: Some(
+                                conf.table_parquet_options().try_into()?,
+                            ),
+                        },
+                    )),
+                });
             }
         }
 
diff --git a/datafusion/substrait/src/physical_plan/producer.rs b/datafusion/substrait/src/physical_plan/producer.rs
index 65eaf79852..9ba0e0c964 100644
--- a/datafusion/substrait/src/physical_plan/producer.rs
+++ b/datafusion/substrait/src/physical_plan/producer.rs
@@ -27,7 +27,7 @@ use datafusion::datasource::source::DataSourceExec;
 use datafusion::error::{DataFusionError, Result};
 use datafusion::physical_plan::{displayable, ExecutionPlan};
 
-use datafusion::datasource::physical_plan::{FileScanConfig, ParquetSource};
+use datafusion::datasource::physical_plan::ParquetSource;
 use substrait::proto::expression::mask_expression::{StructItem, StructSelect};
 use substrait::proto::expression::MaskExpression;
 use substrait::proto::r#type::{
@@ -52,89 +52,82 @@ pub fn to_substrait_rel(
     ),
 ) -> Result<Box<Rel>> {
     if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
-        let data_source = data_source_exec.data_source();
-        if let Some(file_config) = data_source.as_any().downcast_ref::<FileScanConfig>() {
-            let is_parquet = file_config
-                .file_source()
-                .as_any()
-                .downcast_ref::<ParquetSource>()
-                .is_some();
-            if is_parquet {
-                let mut substrait_files = vec![];
-                for (partition_index, files) in file_config.file_groups.iter().enumerate()
-                {
-                    for file in files.iter() {
-                        substrait_files.push(FileOrFiles {
-                            partition_index: partition_index.try_into().unwrap(),
-                            start: 0,
-                            length: file.object_meta.size as u64,
-                            path_type: Some(PathType::UriPath(
-                                file.object_meta.location.as_ref().to_string(),
-                            )),
-                            file_format: Some(FileFormat::Parquet(ParquetReadOptions {})),
-                        });
-                    }
+        if let Some((file_config, _)) =
+            data_source_exec.downcast_to_file_source::<ParquetSource>()
+        {
+            let mut substrait_files = vec![];
+            for (partition_index, files) in file_config.file_groups.iter().enumerate() {
+                for file in files.iter() {
+                    substrait_files.push(FileOrFiles {
+                        partition_index: partition_index.try_into().unwrap(),
+                        start: 0,
+                        length: file.object_meta.size as u64,
+                        path_type: Some(PathType::UriPath(
+                            file.object_meta.location.as_ref().to_string(),
+                        )),
+                        file_format: Some(FileFormat::Parquet(ParquetReadOptions {})),
+                    });
                 }
+            }
 
-                let mut names = vec![];
-                let mut types = vec![];
+            let mut names = vec![];
+            let mut types = vec![];
 
-                for field in file_config.file_schema.fields.iter() {
-                    match to_substrait_type(field.data_type(), field.is_nullable()) {
-                        Ok(t) => {
-                            names.push(field.name().clone());
-                            types.push(t);
-                        }
-                        Err(e) => return Err(e),
+            for field in file_config.file_schema.fields.iter() {
+                match to_substrait_type(field.data_type(), field.is_nullable()) {
+                    Ok(t) => {
+                        names.push(field.name().clone());
+                        types.push(t);
                     }
+                    Err(e) => return Err(e),
                 }
+            }
 
-                let type_info = Struct {
-                    types,
-                    // FIXME: duckdb doesn't set this field, keep it as default variant 0.
-                    // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1106-L1127
-                    type_variation_reference: 0,
-                    nullability: Nullability::Required.into(),
-                };
+            let type_info = Struct {
+                types,
+                // FIXME: duckdb doesn't set this field, keep it as default variant 0.
+                // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1106-L1127
+                type_variation_reference: 0,
+                nullability: Nullability::Required.into(),
+            };
 
-                let mut select_struct = None;
-                if let Some(projection) = file_config.projection.as_ref() {
-                    let struct_items = projection
-                        .iter()
-                        .map(|index| StructItem {
-                            field: *index as i32,
-                            // FIXME: duckdb sets this to None, but it's not clear why.
-                            // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1191
-                            child: None,
-                        })
-                        .collect();
+            let mut select_struct = None;
+            if let Some(projection) = file_config.projection.as_ref() {
+                let struct_items = projection
+                    .iter()
+                    .map(|index| StructItem {
+                        field: *index as i32,
+                        // FIXME: duckdb sets this to None, but it's not clear why.
+                        // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1191
+                        child: None,
+                    })
+                    .collect();
 
-                    select_struct = Some(StructSelect { struct_items });
-                }
+                select_struct = Some(StructSelect { struct_items });
+            }
 
-                return Ok(Box::new(Rel {
-                    rel_type: Some(RelType::Read(Box::new(ReadRel {
-                        common: None,
-                        base_schema: Some(NamedStruct {
-                            names,
-                            r#struct: Some(type_info),
-                        }),
-                        filter: None,
-                        best_effort_filter: None,
-                        projection: Some(MaskExpression {
-                            select: select_struct,
-                            // FIXME: duckdb set this to true, but it's not clear why.
-                            // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1186.
-                            maintain_singular_struct: true,
-                        }),
+            return Ok(Box::new(Rel {
+                rel_type: Some(RelType::Read(Box::new(ReadRel {
+                    common: None,
+                    base_schema: Some(NamedStruct {
+                        names,
+                        r#struct: Some(type_info),
+                    }),
+                    filter: None,
+                    best_effort_filter: None,
+                    projection: Some(MaskExpression {
+                        select: select_struct,
+                        // FIXME: duckdb set this to true, but it's not clear why.
+                        // https://github.com/duckdb/substrait/blob/b6f56643cb11d52de0e32c24a01dfd5947df62be/src/to_substrait.cpp#L1186.
+                        maintain_singular_struct: true,
+                    }),
+                    advanced_extension: None,
+                    read_type: Some(ReadType::LocalFiles(LocalFiles {
+                        items: substrait_files,
                         advanced_extension: None,
-                        read_type: Some(ReadType::LocalFiles(LocalFiles {
-                            items: substrait_files,
-                            advanced_extension: None,
-                        })),
-                    }))),
-                }));
-            }
+                    })),
+                }))),
+            }));
         }
     }
     Err(DataFusionError::Substrait(format!(


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org