You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/11/19 10:22:32 UTC
[arrow-datafusion] branch master updated: rename (#4284)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 880e6fc2f rename (#4284)
880e6fc2f is described below
commit 880e6fc2faa0d1974a4af9c9a4585b79b6cd500b
Author: jakevin <ja...@gmail.com>
AuthorDate: Sat Nov 19 18:22:27 2022 +0800
rename (#4284)
---
datafusion/expr/src/logical_plan/builder.rs | 23 +++----
datafusion/expr/src/logical_plan/plan.rs | 21 ++++---
datafusion/expr/src/utils.rs | 8 +--
.../optimizer/src/common_subexpr_eliminate.rs | 15 ++---
datafusion/optimizer/src/limit_push_down.rs | 24 ++++----
datafusion/optimizer/src/projection_push_down.rs | 23 +++----
.../optimizer/src/propagate_empty_relation.rs | 1 -
.../optimizer/src/single_distinct_to_groupby.rs | 1 -
datafusion/optimizer/src/type_coercion.rs | 71 ++++++++--------------
9 files changed, 88 insertions(+), 99 deletions(-)
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 1bbaf6da9..e9ef0f0cf 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -410,7 +410,6 @@ impl LogicalPlanBuilder {
Ok(Self::from(LogicalPlan::Projection(Projection::try_new(
new_expr,
Arc::new(sort_plan),
- None,
)?)))
}
@@ -874,12 +873,12 @@ pub fn project_with_column_index_alias(
x => x.alias(schema.field(i).name()),
})
.collect::<Vec<_>>();
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- alias_expr, input, schema, alias,
- )?))
+ Ok(LogicalPlan::Projection(
+ Projection::try_new_with_schema_alias(alias_expr, input, schema, alias)?,
+ ))
}
-/// Union two logical plans with an optional alias.
+/// Union two logical plans.
pub fn union(left_plan: LogicalPlan, right_plan: LogicalPlan) -> Result<LogicalPlan> {
let left_col_num = left_plan.schema().fields().len();
@@ -986,12 +985,14 @@ pub fn project_with_alias(
None => input_schema,
};
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- projected_expr,
- Arc::new(plan.clone()),
- DFSchemaRef::new(schema),
- alias,
- )?))
+ Ok(LogicalPlan::Projection(
+ Projection::try_new_with_schema_alias(
+ projected_expr,
+ Arc::new(plan.clone()),
+ DFSchemaRef::new(schema),
+ alias,
+ )?,
+ ))
}
/// Create a LogicalPlanBuilder representing a scan of a table with the provided name and schema.
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index 341e879fa..ae4cdd363 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -1118,13 +1118,12 @@ impl Projection {
pub fn try_new(
expr: Vec<Expr>,
input: Arc<LogicalPlan>,
- alias: Option<String>,
) -> Result<Self, DataFusionError> {
let schema = Arc::new(DFSchema::new_with_metadata(
exprlist_to_fields(&expr, &input)?,
input.schema().metadata().clone(),
)?);
- Self::try_new_with_schema(expr, input, schema, alias)
+ Self::try_new_with_schema(expr, input, schema)
}
/// Create a new Projection using the specified output schema
@@ -1132,6 +1131,15 @@ impl Projection {
expr: Vec<Expr>,
input: Arc<LogicalPlan>,
schema: DFSchemaRef,
+ ) -> Result<Self, DataFusionError> {
+ Self::try_new_with_schema_alias(expr, input, schema, None)
+ }
+
+ /// Create a new Projection using the specified output schema
+ pub fn try_new_with_schema_alias(
+ expr: Vec<Expr>,
+ input: Arc<LogicalPlan>,
+ schema: DFSchemaRef,
alias: Option<String>,
) -> Result<Self, DataFusionError> {
if expr.len() != schema.fields().len() {
@@ -1146,11 +1154,7 @@ impl Projection {
}
/// Create a new Projection using the specified output schema
- pub fn new_from_schema(
- input: Arc<LogicalPlan>,
- schema: DFSchemaRef,
- alias: Option<String>,
- ) -> Self {
+ pub fn new_from_schema(input: Arc<LogicalPlan>, schema: DFSchemaRef) -> Self {
let expr: Vec<Expr> = schema
.fields()
.iter()
@@ -1161,7 +1165,7 @@ impl Projection {
expr,
input,
schema,
- alias,
+ alias: None,
}
}
@@ -1990,7 +1994,6 @@ mod tests {
schema: empty_schema.clone(),
})),
empty_schema,
- None,
);
assert_eq!("Error during planning: Projection has mismatch between number of expressions (1) and number of fields in schema (0)", format!("{}", p.err().unwrap()));
Ok(())
diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs
index 007d7366e..341ae928b 100644
--- a/datafusion/expr/src/utils.rs
+++ b/datafusion/expr/src/utils.rs
@@ -366,14 +366,14 @@ pub fn from_plan(
inputs: &[LogicalPlan],
) -> Result<LogicalPlan> {
match plan {
- LogicalPlan::Projection(Projection { schema, alias, .. }) => {
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
+ LogicalPlan::Projection(Projection { schema, alias, .. }) => Ok(
+ LogicalPlan::Projection(Projection::try_new_with_schema_alias(
expr.to_vec(),
Arc::new(inputs[0].clone()),
schema.clone(),
alias.clone(),
- )?))
- }
+ )?),
+ ),
LogicalPlan::Values(Values { schema, .. }) => Ok(LogicalPlan::Values(Values {
schema: schema.clone(),
values: expr
diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs
index 0ba976c08..fdbda47f2 100644
--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs
+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs
@@ -114,12 +114,14 @@ impl OptimizerRule for CommonSubexprEliminate {
optimizer_config,
)?;
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- pop_expr(&mut new_expr)?,
- Arc::new(new_input),
- schema.clone(),
- alias.clone(),
- )?))
+ Ok(LogicalPlan::Projection(
+ Projection::try_new_with_schema_alias(
+ pop_expr(&mut new_expr)?,
+ Arc::new(new_input),
+ schema.clone(),
+ alias.clone(),
+ )?,
+ ))
}
LogicalPlan::Filter(filter) => {
let input = filter.input();
@@ -328,7 +330,6 @@ fn build_project_plan(
project_exprs,
Arc::new(input),
Arc::new(schema),
- None,
)?))
}
diff --git a/datafusion/optimizer/src/limit_push_down.rs b/datafusion/optimizer/src/limit_push_down.rs
index 51b2bb4f0..3cda94b68 100644
--- a/datafusion/optimizer/src/limit_push_down.rs
+++ b/datafusion/optimizer/src/limit_push_down.rs
@@ -160,17 +160,19 @@ fn limit_push_down(
ancestor,
) => {
// Push down limit directly (projection doesn't change number of rows)
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- expr.clone(),
- Arc::new(limit_push_down(
- _optimizer,
- ancestor,
- input.as_ref(),
- _optimizer_config,
- )?),
- schema.clone(),
- alias.clone(),
- )?))
+ Ok(LogicalPlan::Projection(
+ Projection::try_new_with_schema_alias(
+ expr.clone(),
+ Arc::new(limit_push_down(
+ _optimizer,
+ ancestor,
+ input.as_ref(),
+ _optimizer_config,
+ )?),
+ schema.clone(),
+ alias.clone(),
+ )?,
+ ))
}
(
LogicalPlan::Union(Union { inputs, schema }),
diff --git a/datafusion/optimizer/src/projection_push_down.rs b/datafusion/optimizer/src/projection_push_down.rs
index 874e042e4..5a44247ea 100644
--- a/datafusion/optimizer/src/projection_push_down.rs
+++ b/datafusion/optimizer/src/projection_push_down.rs
@@ -192,12 +192,16 @@ fn optimize_plan(
Ok(new_input)
} else {
let metadata = new_input.schema().metadata().clone();
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- new_expr,
- Arc::new(new_input),
- DFSchemaRef::new(DFSchema::new_with_metadata(new_fields, metadata)?),
- alias.clone(),
- )?))
+ Ok(LogicalPlan::Projection(
+ Projection::try_new_with_schema_alias(
+ new_expr,
+ Arc::new(new_input),
+ DFSchemaRef::new(DFSchema::new_with_metadata(
+ new_fields, metadata,
+ )?),
+ alias.clone(),
+ )?,
+ ))
}
}
LogicalPlan::Join(Join {
@@ -836,11 +840,8 @@ mod tests {
// that the Column references are unqualified (e.g. their
// relation is `None`). PlanBuilder resolves the expressions
let expr = vec![col("a"), col("b")];
- let plan = LogicalPlan::Projection(Projection::try_new(
- expr,
- Arc::new(table_scan),
- None,
- )?);
+ let plan =
+ LogicalPlan::Projection(Projection::try_new(expr, Arc::new(table_scan))?);
assert_fields_eq(&plan, vec!["a", "b"]);
diff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rs
index c1a95c8d3..2de7e72ef 100644
--- a/datafusion/optimizer/src/propagate_empty_relation.rs
+++ b/datafusion/optimizer/src/propagate_empty_relation.rs
@@ -120,7 +120,6 @@ impl OptimizerRule for PropagateEmptyRelation {
Ok(LogicalPlan::Projection(Projection::new_from_schema(
Arc::new(child),
optimized_children_plan.schema().clone(),
- None,
)))
}
} else {
diff --git a/datafusion/optimizer/src/single_distinct_to_groupby.rs b/datafusion/optimizer/src/single_distinct_to_groupby.rs
index 232003371..13cf6d3a8 100644
--- a/datafusion/optimizer/src/single_distinct_to_groupby.rs
+++ b/datafusion/optimizer/src/single_distinct_to_groupby.rs
@@ -196,7 +196,6 @@ impl OptimizerRule for SingleDistinctToGroupBy {
alias_expr,
Arc::new(outer_aggr),
schema.clone(),
- None,
)?))
} else {
utils::optimize_children(self, plan, _optimizer_config)
diff --git a/datafusion/optimizer/src/type_coercion.rs b/datafusion/optimizer/src/type_coercion.rs
index 4e3ea1724..266f557cb 100644
--- a/datafusion/optimizer/src/type_coercion.rs
+++ b/datafusion/optimizer/src/type_coercion.rs
@@ -613,7 +613,7 @@ mod test {
.unwrap(),
),
}));
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -640,7 +640,6 @@ mod test {
let plan = LogicalPlan::Projection(Projection::try_new(
vec![expr.clone().or(expr)],
empty,
- None,
)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
@@ -669,7 +668,7 @@ mod test {
)),
args: vec![lit(123_i32)],
};
- let plan = LogicalPlan::Projection(Projection::try_new(vec![udf], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![udf], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -695,7 +694,7 @@ mod test {
)),
args: vec![lit("Apple")],
};
- let plan = LogicalPlan::Projection(Projection::try_new(vec![udf], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![udf], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).err().unwrap();
@@ -718,7 +717,6 @@ mod test {
let plan = LogicalPlan::Projection(Projection::try_new(
vec![scalar_function_expr],
empty,
- None,
)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
@@ -746,7 +744,7 @@ mod test {
args: vec![lit(10i64)],
filter: None,
};
- let plan = LogicalPlan::Projection(Projection::try_new(vec![udaf], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![udaf], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -778,7 +776,7 @@ mod test {
args: vec![lit("10")],
filter: None,
};
- let plan = LogicalPlan::Projection(Projection::try_new(vec![udaf], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![udaf], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config);
@@ -800,8 +798,7 @@ mod test {
distinct: false,
filter: None,
};
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![agg_expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![agg_expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -818,8 +815,7 @@ mod test {
distinct: false,
filter: None,
};
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![agg_expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![agg_expr], empty)?);
let plan = rule.optimize(&plan, &mut config)?;
assert_eq!(
"Projection: AVG(a)\n EmptyRelation",
@@ -838,7 +834,7 @@ mod test {
distinct: false,
filter: None,
};
- let expr = Projection::try_new(vec![agg_expr], empty, None);
+ let expr = Projection::try_new(vec![agg_expr], empty);
assert!(expr.is_err());
assert_eq!(
"Plan(\"The function Avg does not support inputs of type Utf8.\")",
@@ -856,7 +852,7 @@ mod test {
produce_one_row: false,
schema: Arc::new(DFSchema::empty()),
}));
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -881,7 +877,7 @@ mod test {
.unwrap(),
),
}));
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config)?;
@@ -902,7 +898,7 @@ mod test {
.unwrap(),
),
}));
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let plan = rule.optimize(&plan, &mut config)?;
assert_eq!(
"Projection: CAST(a AS Decimal128(24, 4)) IN ([CAST(Int32(1) AS Decimal128(24, 4)), CAST(Int8(4) AS Decimal128(24, 4)), CAST(Int64(8) AS Decimal128(24, 4))]) AS a IN (Map { iter: Iter([Int32(1), Int8(4), Int64(8)]) })\
@@ -917,11 +913,8 @@ mod test {
// is true
let expr = col("a").is_true();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(
- vec![expr.clone()],
- empty,
- None,
- )?);
+ let plan =
+ LogicalPlan::Projection(Projection::try_new(vec![expr.clone()], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -930,7 +923,7 @@ mod test {
&format!("{:?}", plan)
);
let empty = empty_with_type(DataType::Int64);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let plan = rule.optimize(&plan, &mut config);
assert!(plan.is_err());
assert!(plan.unwrap_err().to_string().contains("'Int64 IS DISTINCT FROM Boolean' can't be evaluated because there isn't a common type to coerce the types to"));
@@ -938,7 +931,7 @@ mod test {
// is not true
let expr = col("a").is_not_true();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let plan = rule.optimize(&plan, &mut config).unwrap();
assert_eq!(
"Projection: a IS NOT TRUE\n EmptyRelation",
@@ -948,7 +941,7 @@ mod test {
// is false
let expr = col("a").is_false();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let plan = rule.optimize(&plan, &mut config).unwrap();
assert_eq!(
"Projection: a IS FALSE\n EmptyRelation",
@@ -958,7 +951,7 @@ mod test {
// is not false
let expr = col("a").is_not_false();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let plan = rule.optimize(&plan, &mut config).unwrap();
assert_eq!(
"Projection: a IS NOT FALSE\n EmptyRelation",
@@ -974,8 +967,7 @@ mod test {
let pattern = Box::new(lit(ScalarValue::new_utf8("abc")));
let like_expr = Expr::Like(Like::new(false, expr, pattern, None));
let empty = empty_with_type(DataType::Utf8);
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -988,8 +980,7 @@ mod test {
let pattern = Box::new(lit(ScalarValue::Null));
let like_expr = Expr::Like(Like::new(false, expr, pattern, None));
let empty = empty_with_type(DataType::Utf8);
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -1003,8 +994,7 @@ mod test {
let pattern = Box::new(lit(ScalarValue::new_utf8("abc")));
let like_expr = Expr::Like(Like::new(false, expr, pattern, None));
let empty = empty_with_type(DataType::Int64);
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![like_expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config);
@@ -1018,11 +1008,8 @@ mod test {
// unknown
let expr = col("a").is_unknown();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(
- vec![expr.clone()],
- empty,
- None,
- )?);
+ let plan =
+ LogicalPlan::Projection(Projection::try_new(vec![expr.clone()], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -1032,7 +1019,7 @@ mod test {
);
let empty = empty_with_type(DataType::Utf8);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config);
@@ -1042,7 +1029,7 @@ mod test {
// is not unknown
let expr = col("a").is_not_unknown();
let empty = empty_with_type(DataType::Boolean);
- let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -1062,11 +1049,8 @@ mod test {
{
let expr = concat(&args);
- let plan = LogicalPlan::Projection(Projection::try_new(
- vec![expr],
- empty.clone(),
- None,
- )?);
+ let plan =
+ LogicalPlan::Projection(Projection::try_new(vec![expr], empty.clone())?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();
@@ -1080,8 +1064,7 @@ mod test {
{
let expr = concat_ws(lit("-"), args.to_vec());
- let plan =
- LogicalPlan::Projection(Projection::try_new(vec![expr], empty, None)?);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
let rule = TypeCoercion::new();
let mut config = OptimizerConfig::default();
let plan = rule.optimize(&plan, &mut config).unwrap();