You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/05/16 15:00:38 UTC
[arrow-datafusion] branch master updated: Remove scan_avro methods from LogicalPlanBuilder (#2540)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 716e0bcfe Remove scan_avro methods from LogicalPlanBuilder (#2540)
716e0bcfe is described below
commit 716e0bcfe1bdcd8fcc842001d1bf30673cc26e8f
Author: Andy Grove <ag...@apache.org>
AuthorDate: Mon May 16 09:00:33 2022 -0600
Remove scan_avro methods from LogicalPlanBuilder (#2540)
---
datafusion/core/src/execution/context.rs | 30 +++++++++++-------
datafusion/core/src/logical_plan/builder.rs | 49 -----------------------------
2 files changed, 18 insertions(+), 61 deletions(-)
diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs
index 202928a0c..394d03755 100644
--- a/datafusion/core/src/execution/context.rs
+++ b/datafusion/core/src/execution/context.rs
@@ -504,18 +504,24 @@ impl SessionContext {
let uri: String = uri.into();
let (object_store, path) = self.runtime_env().object_store(&uri)?;
let target_partitions = self.copied_config().target_partitions;
- Ok(Arc::new(DataFrame::new(
- self.state.clone(),
- &LogicalPlanBuilder::scan_avro(
- object_store,
- path,
- options,
- None,
- target_partitions,
- )
- .await?
- .build()?,
- )))
+
+ let listing_options = options.to_listing_options(target_partitions);
+
+ let path: String = path.into();
+
+ let resolved_schema = match options.schema {
+ Some(s) => s,
+ None => {
+ listing_options
+ .infer_schema(Arc::clone(&object_store), &path)
+ .await?
+ }
+ };
+ let config = ListingTableConfig::new(object_store, path.clone())
+ .with_listing_options(listing_options)
+ .with_schema(resolved_schema);
+ let provider = ListingTable::try_new(config)?;
+ self.read_table(Arc::new(provider))
}
/// Creates a DataFrame for reading an Json data source.
diff --git a/datafusion/core/src/logical_plan/builder.rs b/datafusion/core/src/logical_plan/builder.rs
index f01ef9489..0b419b840 100644
--- a/datafusion/core/src/logical_plan/builder.rs
+++ b/datafusion/core/src/logical_plan/builder.rs
@@ -256,55 +256,6 @@ impl LogicalPlanBuilder {
Self::scan(table_name, Arc::new(provider), projection)
}
- /// Scan an Avro data source
- pub async fn scan_avro(
- object_store: Arc<dyn ObjectStore>,
- path: impl Into<String>,
- options: AvroReadOptions<'_>,
- projection: Option<Vec<usize>>,
- target_partitions: usize,
- ) -> Result<Self> {
- let path = path.into();
- Self::scan_avro_with_name(
- object_store,
- path.clone(),
- options,
- projection,
- path,
- target_partitions,
- )
- .await
- }
-
- /// Scan an Avro data source and register it with a given table name
- pub async fn scan_avro_with_name(
- object_store: Arc<dyn ObjectStore>,
- path: impl Into<String>,
- options: AvroReadOptions<'_>,
- projection: Option<Vec<usize>>,
- table_name: impl Into<String>,
- target_partitions: usize,
- ) -> Result<Self> {
- let listing_options = options.to_listing_options(target_partitions);
-
- let path: String = path.into();
-
- let resolved_schema = match options.schema {
- Some(s) => s,
- None => {
- listing_options
- .infer_schema(Arc::clone(&object_store), &path)
- .await?
- }
- };
- let config = ListingTableConfig::new(object_store, path)
- .with_listing_options(listing_options)
- .with_schema(resolved_schema);
- let provider = ListingTable::try_new(config)?;
-
- Self::scan(table_name, Arc::new(provider), projection)
- }
-
/// Scan an empty data source, mainly used in tests
pub fn scan_empty(
name: Option<&str>,