You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/04/20 21:33:14 UTC
[arrow-datafusion] branch master updated: Move FileType enum from sql module to logical_plan module (#2290)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 8e5551fc0 Move FileType enum from sql module to logical_plan module (#2290)
8e5551fc0 is described below
commit 8e5551fc03ac174938cc7be67a1382d2262484b6
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Apr 20 15:33:09 2022 -0600
Move FileType enum from sql module to logical_plan module (#2290)
---
ballista/rust/client/src/context.rs | 4 ++--
ballista/rust/core/src/serde/logical_plan/from_proto.rs | 6 +++---
ballista/rust/core/src/serde/logical_plan/mod.rs | 7 +++----
datafusion/core/src/execution/context.rs | 5 +++--
datafusion/core/src/logical_plan/mod.rs | 5 +++--
datafusion/core/src/logical_plan/plan.rs | 14 +++++++++++++-
datafusion/core/src/sql/parser.rs | 14 +-------------
datafusion/core/src/sql/planner.rs | 6 +++---
8 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
index 7dc7ec63b..7d21b1121 100644
--- a/ballista/rust/client/src/context.rs
+++ b/ballista/rust/client/src/context.rs
@@ -34,11 +34,11 @@ use datafusion::catalog::TableReference;
use datafusion::dataframe::DataFrame;
use datafusion::datasource::TableProvider;
use datafusion::error::{DataFusionError, Result};
-use datafusion::logical_plan::{CreateExternalTable, LogicalPlan, TableScan};
+use datafusion::logical_plan::{CreateExternalTable, FileType, LogicalPlan, TableScan};
use datafusion::prelude::{
AvroReadOptions, CsvReadOptions, ParquetReadOptions, SessionConfig, SessionContext,
};
-use datafusion::sql::parser::{DFParser, FileType, Statement as DFStatement};
+use datafusion::sql::parser::{DFParser, Statement as DFStatement};
struct BallistaContextState {
/// Ballista configuration
diff --git a/ballista/rust/core/src/serde/logical_plan/from_proto.rs b/ballista/rust/core/src/serde/logical_plan/from_proto.rs
index afd9b268d..a17c646ca 100644
--- a/ballista/rust/core/src/serde/logical_plan/from_proto.rs
+++ b/ballista/rust/core/src/serde/logical_plan/from_proto.rs
@@ -39,9 +39,9 @@ impl TryFrom<i32> for protobuf::FileType {
}
#[allow(clippy::from_over_into)]
-impl Into<datafusion::sql::parser::FileType> for protobuf::FileType {
- fn into(self) -> datafusion::sql::parser::FileType {
- use datafusion::sql::parser::FileType;
+impl Into<datafusion::logical_plan::FileType> for protobuf::FileType {
+ fn into(self) -> datafusion::logical_plan::FileType {
+ use datafusion::logical_plan::FileType;
match self {
protobuf::FileType::NdJson => FileType::NdJson,
protobuf::FileType::Parquet => FileType::Parquet,
diff --git a/ballista/rust/core/src/serde/logical_plan/mod.rs b/ballista/rust/core/src/serde/logical_plan/mod.rs
index a0264271a..d88498b86 100644
--- a/ballista/rust/core/src/serde/logical_plan/mod.rs
+++ b/ballista/rust/core/src/serde/logical_plan/mod.rs
@@ -818,7 +818,7 @@ impl AsLogicalPlan for LogicalPlanNode {
table_partition_cols,
if_not_exists,
}) => {
- use datafusion::sql::parser::FileType;
+ use datafusion::logical_plan::FileType;
let pb_file_type: protobuf::FileType = match file_type {
FileType::NdJson => protobuf::FileType::NdJson,
@@ -994,11 +994,10 @@ mod roundtrip_tests {
},
datasource::listing::ListingTable,
logical_plan::{
- col, CreateExternalTable, Expr, LogicalPlan, LogicalPlanBuilder, Repartition,
- ToDFSchema,
+ col, CreateExternalTable, Expr, FileType, LogicalPlan, LogicalPlanBuilder,
+ Repartition, ToDFSchema,
},
prelude::*,
- sql::parser::FileType,
};
use std::io;
use std::sync::Arc;
diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs
index f7262def0..eae531970 100644
--- a/datafusion/core/src/execution/context.rs
+++ b/datafusion/core/src/execution/context.rs
@@ -62,7 +62,8 @@ use crate::datasource::TableProvider;
use crate::error::{DataFusionError, Result};
use crate::logical_plan::{
CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateMemoryTable,
- DropTable, FunctionRegistry, LogicalPlan, LogicalPlanBuilder, UNNAMED_TABLE,
+ DropTable, FileType, FunctionRegistry, LogicalPlan, LogicalPlanBuilder,
+ UNNAMED_TABLE,
};
use crate::optimizer::common_subexpr_eliminate::CommonSubexprEliminate;
use crate::optimizer::filter_push_down::FilterPushDown;
@@ -86,7 +87,7 @@ use crate::physical_plan::udf::ScalarUDF;
use crate::physical_plan::ExecutionPlan;
use crate::physical_plan::PhysicalPlanner;
use crate::sql::{
- parser::{DFParser, FileType},
+ parser::DFParser,
planner::{ContextProvider, SqlToRel},
};
use crate::variable::{VarProvider, VarType};
diff --git a/datafusion/core/src/logical_plan/mod.rs b/datafusion/core/src/logical_plan/mod.rs
index cc5023008..d2c0a1fa3 100644
--- a/datafusion/core/src/logical_plan/mod.rs
+++ b/datafusion/core/src/logical_plan/mod.rs
@@ -64,8 +64,9 @@ pub use extension::UserDefinedLogicalNode;
pub use operators::Operator;
pub use plan::{
CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateMemoryTable,
- CrossJoin, DropTable, EmptyRelation, JoinConstraint, JoinType, Limit, LogicalPlan,
- Partitioning, PlanType, PlanVisitor, Repartition, TableScan, Union, Values,
+ CrossJoin, DropTable, EmptyRelation, FileType, JoinConstraint, JoinType, Limit,
+ LogicalPlan, Partitioning, PlanType, PlanVisitor, Repartition, TableScan, Union,
+ Values,
};
pub(crate) use plan::{StringifiedPlan, ToStringifiedPlan};
pub use registry::FunctionRegistry;
diff --git a/datafusion/core/src/logical_plan/plan.rs b/datafusion/core/src/logical_plan/plan.rs
index 66307c6ab..72fd3fa28 100644
--- a/datafusion/core/src/logical_plan/plan.rs
+++ b/datafusion/core/src/logical_plan/plan.rs
@@ -24,7 +24,6 @@ use crate::datasource::datasource::TableProviderFilterPushDown;
use crate::datasource::TableProvider;
use crate::error::DataFusionError;
use crate::logical_plan::dfschema::DFSchemaRef;
-use crate::sql::parser::FileType;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use std::fmt::Formatter;
use std::{
@@ -184,6 +183,19 @@ pub struct CreateMemoryTable {
pub if_not_exists: bool,
}
+/// Types of files to parse as DataFrames
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum FileType {
+ /// Newline-delimited JSON
+ NdJson,
+ /// Apache Parquet columnar storage
+ Parquet,
+ /// Comma separated values
+ CSV,
+ /// Avro binary records
+ Avro,
+}
+
/// Creates an external table.
#[derive(Clone)]
pub struct CreateExternalTable {
diff --git a/datafusion/core/src/sql/parser.rs b/datafusion/core/src/sql/parser.rs
index 2b98a3090..0f4596fe6 100644
--- a/datafusion/core/src/sql/parser.rs
+++ b/datafusion/core/src/sql/parser.rs
@@ -19,6 +19,7 @@
//!
//! Declares a SQL parser based on sqlparser that handles custom formats that we need.
+use crate::logical_plan::FileType;
use sqlparser::{
ast::{ColumnDef, ColumnOptionDef, Statement as SQLStatement, TableConstraint},
dialect::{keywords::Keyword, Dialect, GenericDialect},
@@ -35,19 +36,6 @@ macro_rules! parser_err {
};
}
-/// Types of files to parse as DataFrames
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum FileType {
- /// Newline-delimited JSON
- NdJson,
- /// Apache Parquet columnar storage
- Parquet,
- /// Comma separated values
- CSV,
- /// Avro binary records
- Avro,
-}
-
impl FromStr for FileType {
type Err = ParserError;
diff --git a/datafusion/core/src/sql/planner.rs b/datafusion/core/src/sql/planner.rs
index 2f2bdc5fe..a60f8c2f8 100644
--- a/datafusion/core/src/sql/planner.rs
+++ b/datafusion/core/src/sql/planner.rs
@@ -31,8 +31,8 @@ use crate::logical_plan::{
and, builder::expand_qualified_wildcard, builder::expand_wildcard, col, lit,
normalize_col, union_with_alias, Column, CreateCatalog, CreateCatalogSchema,
CreateExternalTable as PlanCreateExternalTable, CreateMemoryTable, DFSchema,
- DFSchemaRef, DropTable, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType,
- ToDFSchema, ToStringifiedPlan,
+ DFSchemaRef, DropTable, Expr, FileType, LogicalPlan, LogicalPlanBuilder, Operator,
+ PlanType, ToDFSchema, ToStringifiedPlan,
};
use crate::optimizer::utils::exprlist_to_columns;
use crate::prelude::JoinType;
@@ -43,7 +43,7 @@ use crate::{
physical_plan::aggregates,
physical_plan::udaf::AggregateUDF,
physical_plan::udf::ScalarUDF,
- sql::parser::{CreateExternalTable, FileType, Statement as DFStatement},
+ sql::parser::{CreateExternalTable, Statement as DFStatement},
};
use arrow::datatypes::*;
use datafusion_expr::{window_function::WindowFunction, BuiltinScalarFunction};