You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by al...@apache.org on 2024/07/05 11:49:09 UTC
(datafusion) branch main updated: Remove outdated `rewrite_expr.rs` example (#11085)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 351e5f9564 Remove outdated `rewrite_expr.rs` example (#11085)
351e5f9564 is described below
commit 351e5f95646035a8f742811af0a61d00480fefc2
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Jul 5 07:49:06 2024 -0400
Remove outdated `rewrite_expr.rs` example (#11085)
* Remove outdated rewrite_expr.rs example
* Update docs
---
datafusion-examples/README.md | 1 -
datafusion-examples/examples/rewrite_expr.rs | 251 ---------------------
datafusion/optimizer/README.md | 6 +-
.../library-user-guide/working-with-exprs.md | 6 +-
4 files changed, 9 insertions(+), 255 deletions(-)
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index a208eee135..dc92019035 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -72,7 +72,6 @@ cargo run --example dataframe
- [`query-aws-s3.rs`](examples/external_dependency/query-aws-s3.rs): Configure `object_store` and run a query against files stored in AWS S3
- [`query-http-csv.rs`](examples/query-http-csv.rs): Configure `object_store` and run a query against files vi HTTP
- [`regexp.rs`](examples/regexp.rs): Examples of using regular expression functions
-- [`rewrite_expr.rs`](examples/rewrite_expr.rs): Define and invoke a custom Query Optimizer pass
- [`simple_udaf.rs`](examples/simple_udaf.rs): Define and invoke a User Defined Aggregate Function (UDAF)
- [`simple_udf.rs`](examples/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)
- [`simple_udfw.rs`](examples/simple_udwf.rs): Define and invoke a User Defined Window Function (UDWF)
diff --git a/datafusion-examples/examples/rewrite_expr.rs b/datafusion-examples/examples/rewrite_expr.rs
deleted file mode 100644
index 06286d5d66..0000000000
--- a/datafusion-examples/examples/rewrite_expr.rs
+++ /dev/null
@@ -1,251 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion_common::{plan_err, DataFusionError, Result, ScalarValue};
-use datafusion_expr::{
- AggregateUDF, Between, Expr, Filter, LogicalPlan, ScalarUDF, TableSource, WindowUDF,
-};
-use datafusion_optimizer::analyzer::{Analyzer, AnalyzerRule};
-use datafusion_optimizer::optimizer::{ApplyOrder, Optimizer};
-use datafusion_optimizer::{OptimizerConfig, OptimizerContext, OptimizerRule};
-use datafusion_sql::planner::{ContextProvider, SqlToRel};
-use datafusion_sql::sqlparser::dialect::PostgreSqlDialect;
-use datafusion_sql::sqlparser::parser::Parser;
-use datafusion_sql::TableReference;
-use std::any::Any;
-use std::sync::Arc;
-
-pub fn main() -> Result<()> {
- // produce a logical plan using the datafusion-sql crate
- let dialect = PostgreSqlDialect {};
- let sql = "SELECT * FROM person WHERE age BETWEEN 21 AND 32";
- let statements = Parser::parse_sql(&dialect, sql)?;
-
- // produce a logical plan using the datafusion-sql crate
- let context_provider = MyContextProvider::default();
- let sql_to_rel = SqlToRel::new(&context_provider);
- let logical_plan = sql_to_rel.sql_statement_to_plan(statements[0].clone())?;
- println!(
- "Unoptimized Logical Plan:\n\n{}\n",
- logical_plan.display_indent()
- );
-
- // run the analyzer with our custom rule
- let config = OptimizerContext::default().with_skip_failing_rules(false);
- let analyzer = Analyzer::with_rules(vec![Arc::new(MyAnalyzerRule {})]);
- let analyzed_plan =
- analyzer.execute_and_check(logical_plan, config.options(), |_, _| {})?;
- println!(
- "Analyzed Logical Plan:\n\n{}\n",
- analyzed_plan.display_indent()
- );
-
- // then run the optimizer with our custom rule
- let optimizer = Optimizer::with_rules(vec![Arc::new(MyOptimizerRule {})]);
- let optimized_plan = optimizer.optimize(analyzed_plan, &config, observe)?;
- println!(
- "Optimized Logical Plan:\n\n{}\n",
- optimized_plan.display_indent()
- );
-
- Ok(())
-}
-
-fn observe(plan: &LogicalPlan, rule: &dyn OptimizerRule) {
- println!(
- "After applying rule '{}':\n{}\n",
- rule.name(),
- plan.display_indent()
- )
-}
-
-/// An example analyzer rule that changes Int64 literals to UInt64
-struct MyAnalyzerRule {}
-
-impl AnalyzerRule for MyAnalyzerRule {
- fn analyze(&self, plan: LogicalPlan, _config: &ConfigOptions) -> Result<LogicalPlan> {
- Self::analyze_plan(plan)
- }
-
- fn name(&self) -> &str {
- "my_analyzer_rule"
- }
-}
-
-impl MyAnalyzerRule {
- fn analyze_plan(plan: LogicalPlan) -> Result<LogicalPlan> {
- plan.transform(|plan| {
- Ok(match plan {
- LogicalPlan::Filter(filter) => {
- let predicate = Self::analyze_expr(filter.predicate.clone())?;
- Transformed::yes(LogicalPlan::Filter(Filter::try_new(
- predicate,
- filter.input,
- )?))
- }
- _ => Transformed::no(plan),
- })
- })
- .data()
- }
-
- fn analyze_expr(expr: Expr) -> Result<Expr> {
- expr.transform(|expr| {
- // closure is invoked for all sub expressions
- Ok(match expr {
- Expr::Literal(ScalarValue::Int64(i)) => {
- // transform to UInt64
- Transformed::yes(Expr::Literal(ScalarValue::UInt64(
- i.map(|i| i as u64),
- )))
- }
- _ => Transformed::no(expr),
- })
- })
- .data()
- }
-}
-
-/// An example optimizer rule that rewrite BETWEEN expression to binary compare expressions
-struct MyOptimizerRule {}
-
-impl OptimizerRule for MyOptimizerRule {
- fn name(&self) -> &str {
- "my_optimizer_rule"
- }
-
- fn apply_order(&self) -> Option<ApplyOrder> {
- Some(ApplyOrder::BottomUp)
- }
-
- fn supports_rewrite(&self) -> bool {
- true
- }
-
- fn rewrite(
- &self,
- plan: LogicalPlan,
- _config: &dyn OptimizerConfig,
- ) -> Result<Transformed<LogicalPlan>, DataFusionError> {
- match plan {
- LogicalPlan::Filter(filter) => {
- let predicate = my_rewrite(filter.predicate.clone())?;
- Ok(Transformed::yes(LogicalPlan::Filter(Filter::try_new(
- predicate,
- filter.input.clone(),
- )?)))
- }
- _ => Ok(Transformed::no(plan)),
- }
- }
-}
-
-/// use rewrite_expr to modify the expression tree.
-fn my_rewrite(expr: Expr) -> Result<Expr> {
- expr.transform(|expr| {
- // closure is invoked for all sub expressions
- Ok(match expr {
- Expr::Between(Between {
- expr,
- negated,
- low,
- high,
- }) => {
- // unbox
- let expr: Expr = *expr;
- let low: Expr = *low;
- let high: Expr = *high;
- if negated {
- Transformed::yes(expr.clone().lt(low).or(expr.gt(high)))
- } else {
- Transformed::yes(expr.clone().gt_eq(low).and(expr.lt_eq(high)))
- }
- }
- _ => Transformed::no(expr),
- })
- })
- .data()
-}
-
-#[derive(Default)]
-struct MyContextProvider {
- options: ConfigOptions,
-}
-
-impl ContextProvider for MyContextProvider {
- fn get_table_source(&self, name: TableReference) -> Result<Arc<dyn TableSource>> {
- if name.table() == "person" {
- Ok(Arc::new(MyTableSource {
- schema: Arc::new(Schema::new(vec![
- Field::new("name", DataType::Utf8, false),
- Field::new("age", DataType::UInt8, false),
- ])),
- }))
- } else {
- plan_err!("table not found")
- }
- }
-
- fn get_function_meta(&self, _name: &str) -> Option<Arc<ScalarUDF>> {
- None
- }
-
- fn get_aggregate_meta(&self, _name: &str) -> Option<Arc<AggregateUDF>> {
- None
- }
-
- fn get_variable_type(&self, _variable_names: &[String]) -> Option<DataType> {
- None
- }
-
- fn get_window_meta(&self, _name: &str) -> Option<Arc<WindowUDF>> {
- None
- }
-
- fn options(&self) -> &ConfigOptions {
- &self.options
- }
-
- fn udf_names(&self) -> Vec<String> {
- Vec::new()
- }
-
- fn udaf_names(&self) -> Vec<String> {
- Vec::new()
- }
-
- fn udwf_names(&self) -> Vec<String> {
- Vec::new()
- }
-}
-
-struct MyTableSource {
- schema: SchemaRef,
-}
-
-impl TableSource for MyTableSource {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn schema(&self) -> SchemaRef {
- self.schema.clone()
- }
-}
diff --git a/datafusion/optimizer/README.md b/datafusion/optimizer/README.md
index 2f1f85e3a5..5aacfaf59c 100644
--- a/datafusion/optimizer/README.md
+++ b/datafusion/optimizer/README.md
@@ -67,8 +67,10 @@ let optimizer = Optimizer::with_rules(vec to learn more about
-the general approach to writing optimizer rules and then move onto studying the existing rules.
+Please refer to the
+[optimizer_rule.rs](../../datafusion-examples/examples/optimizer_rule.rs)
+example to learn more about the general approach to writing optimizer rules and
+then move onto studying the existing rules.
All rules must implement the `OptimizerRule` trait.
diff --git a/docs/source/library-user-guide/working-with-exprs.md b/docs/source/library-user-guide/working-with-exprs.md
index e0c9e69eb6..e0b6f434a0 100644
--- a/docs/source/library-user-guide/working-with-exprs.md
+++ b/docs/source/library-user-guide/working-with-exprs.md
@@ -80,7 +80,11 @@ If you'd like to learn more about `Expr`s, before we get into the details of cre
## Rewriting `Expr`s
-[rewrite_expr.rs](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/rewrite_expr.rs) contains example code for rewriting `Expr`s.
+There are several examples of rewriting and working with `Exprs`:
+
+- [expr_api.rs](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/expr_api.rs)
+- [analyzer_rule.rs](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/analyzer_rule.rs)
+- [optimizer_rule.rs](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/optimizer_rule.rs)
Rewriting Expressions is the process of taking an `Expr` and transforming it into another `Expr`. This is useful for a number of reasons, including:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org