You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/05/16 12:20:38 UTC
[arrow-datafusion] branch master updated: Remove `scan_parquet` methods from `LogicalPlanBuilder` (#2539)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 419addef3 Remove `scan_parquet` methods from `LogicalPlanBuilder` (#2539)
419addef3 is described below
commit 419addef30c486449e1733db1eef03f31252a14f
Author: Andy Grove <ag...@apache.org>
AuthorDate: Mon May 16 06:20:32 2022 -0600
Remove `scan_parquet` methods from `LogicalPlanBuilder` (#2539)
* Remove scan_parquet methods from LogicalPlanBuilder
* simplify code
---
datafusion/core/src/execution/context.rs | 25 +++++++++-------
datafusion/core/src/logical_plan/builder.rs | 45 -----------------------------
2 files changed, 15 insertions(+), 55 deletions(-)
diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs
index 7ca9a1e4d..c18c49eb8 100644
--- a/datafusion/core/src/execution/context.rs
+++ b/datafusion/core/src/execution/context.rs
@@ -581,16 +581,21 @@ impl SessionContext {
let uri: String = uri.into();
let (object_store, path) = self.runtime_env().object_store(&uri)?;
let target_partitions = self.copied_config().target_partitions;
- let logical_plan = LogicalPlanBuilder::scan_parquet(
- object_store,
- path,
- options,
- None,
- target_partitions,
- )
- .await?
- .build()?;
- Ok(Arc::new(DataFrame::new(self.state.clone(), &logical_plan)))
+
+ let listing_options = options.to_listing_options(target_partitions);
+ let path: String = path.into();
+
+ // with parquet we resolve the schema in all cases
+ let resolved_schema = listing_options
+ .infer_schema(Arc::clone(&object_store), &path)
+ .await?;
+
+ let config = ListingTableConfig::new(object_store, path)
+ .with_listing_options(listing_options)
+ .with_schema(resolved_schema);
+
+ let provider = ListingTable::try_new(config)?;
+ self.read_table(Arc::new(provider))
}
/// Creates a DataFrame for reading a custom TableProvider.
diff --git a/datafusion/core/src/logical_plan/builder.rs b/datafusion/core/src/logical_plan/builder.rs
index 69dbde693..c35b522e6 100644
--- a/datafusion/core/src/logical_plan/builder.rs
+++ b/datafusion/core/src/logical_plan/builder.rs
@@ -256,51 +256,6 @@ impl LogicalPlanBuilder {
Self::scan(table_name, Arc::new(provider), projection)
}
- /// Scan a Parquet data source
- pub async fn scan_parquet(
- object_store: Arc<dyn ObjectStore>,
- path: impl Into<String>,
- options: ParquetReadOptions<'_>,
- projection: Option<Vec<usize>>,
- target_partitions: usize,
- ) -> Result<Self> {
- let path = path.into();
- Self::scan_parquet_with_name(
- object_store,
- path.clone(),
- options,
- projection,
- target_partitions,
- path,
- )
- .await
- }
-
- /// Scan a Parquet data source and register it with a given table name
- pub async fn scan_parquet_with_name(
- object_store: Arc<dyn ObjectStore>,
- path: impl Into<String>,
- options: ParquetReadOptions<'_>,
- projection: Option<Vec<usize>>,
- target_partitions: usize,
- table_name: impl Into<String>,
- ) -> Result<Self> {
- let listing_options = options.to_listing_options(target_partitions);
- let path: String = path.into();
-
- // with parquet we resolve the schema in all cases
- let resolved_schema = listing_options
- .infer_schema(Arc::clone(&object_store), &path)
- .await?;
-
- let config = ListingTableConfig::new(object_store, path)
- .with_listing_options(listing_options)
- .with_schema(resolved_schema);
-
- let provider = ListingTable::try_new(config)?;
- Self::scan(table_name, Arc::new(provider), projection)
- }
-
/// Scan an Avro data source
pub async fn scan_avro(
object_store: Arc<dyn ObjectStore>,