You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by gi...@apache.org on 2023/08/25 17:10:54 UTC
[arrow-datafusion] branch asf-site updated: Publish built docs triggered by d059dd32d61df3dbb8e1c7738346aa91a563f120
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/asf-site by this push:
new e76f2bf49a Publish built docs triggered by d059dd32d61df3dbb8e1c7738346aa91a563f120
e76f2bf49a is described below
commit e76f2bf49aa1b6140d0238d0ad1659133de47855
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Fri Aug 25 17:10:49 2023 +0000
Publish built docs triggered by d059dd32d61df3dbb8e1c7738346aa91a563f120
---
.../1a0b5c48023f0bdbcb8dff8324195c3c/expr_api.rs | 134 +++++++++++++
.../97b381d1ef654142806e5e91c3788385/simple_udf.rs | 144 ++++++++++++++
.../simple_udwf.rs | 211 +++++++++++++++++++++
.../simple_udaf.rs | 178 +++++++++++++++++
_sources/library-user-guide/adding-udfs.md.txt | 104 +++++++++-
.../library-user-guide/working-with-exprs.md.txt | 164 +++++++++++++++-
library-user-guide/adding-udfs.html | 135 ++++++++++++-
library-user-guide/working-with-exprs.html | 197 ++++++++++++++++++-
searchindex.js | 2 +-
9 files changed, 1262 insertions(+), 7 deletions(-)
diff --git a/_downloads/1a0b5c48023f0bdbcb8dff8324195c3c/expr_api.rs b/_downloads/1a0b5c48023f0bdbcb8dff8324195c3c/expr_api.rs
new file mode 100644
index 0000000000..97abf4d552
--- /dev/null
+++ b/_downloads/1a0b5c48023f0bdbcb8dff8324195c3c/expr_api.rs
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
+use datafusion::error::Result;
+use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext};
+use datafusion::physical_expr::execution_props::ExecutionProps;
+use datafusion::prelude::*;
+use datafusion_common::{ScalarValue, ToDFSchema};
+use datafusion_expr::expr::BinaryExpr;
+use datafusion_expr::Operator;
+
+/// This example demonstrates the DataFusion [`Expr`] API.
+///
+/// DataFusion comes with a powerful and extensive system for
+/// representing and manipulating expressions such as `A + 5` and `X
+/// IN ('foo', 'bar', 'baz')` and many other constructs.
+#[tokio::main]
+async fn main() -> Result<()> {
+ // The easiest way to do create expressions is to use the
+ // "fluent"-style API, like this:
+ let expr = col("a") + lit(5);
+
+ // this creates the same expression as the following though with
+ // much less code,
+ let expr2 = Expr::BinaryExpr(BinaryExpr::new(
+ Box::new(col("a")),
+ Operator::Plus,
+ Box::new(Expr::Literal(ScalarValue::Int32(Some(5)))),
+ ));
+ assert_eq!(expr, expr2);
+
+ simplify_demo()?;
+
+ Ok(())
+}
+
+/// In addition to easy construction, DataFusion exposes APIs for
+/// working with and simplifying such expressions that call into the
+/// same powerful and extensive implementation used for the query
+/// engine.
+fn simplify_demo() -> Result<()> {
+ // For example, lets say you have has created an expression such
+ // ts = to_timestamp("2020-09-08T12:00:00+00:00")
+ let expr = col("ts").eq(call_fn(
+ "to_timestamp",
+ vec![lit("2020-09-08T12:00:00+00:00")],
+ )?);
+
+ // Naively evaluating such an expression against a large number of
+ // rows would involve re-converting "2020-09-08T12:00:00+00:00" to a
+ // timestamp for each row which gets expensive
+ //
+ // However, DataFusion's simplification logic can do this for you
+
+ // you need to tell DataFusion the type of column "ts":
+ let schema = Schema::new(vec![make_ts_field("ts")]).to_dfschema_ref()?;
+
+ // And then build a simplifier
+ // the ExecutionProps carries information needed to simplify
+ // expressions, such as the current time (to evaluate `now()`
+ // correctly)
+ let props = ExecutionProps::new();
+ let context = SimplifyContext::new(&props).with_schema(schema);
+ let simplifier = ExprSimplifier::new(context);
+
+ // And then call the simplify_expr function:
+ let expr = simplifier.simplify(expr)?;
+
+ // DataFusion has simplified the expression to a comparison with a constant
+ // ts = 1599566400000000000; Tada!
+ assert_eq!(
+ expr,
+ col("ts").eq(lit_timestamp_nano(1599566400000000000i64))
+ );
+
+ // here are some other examples of what DataFusion is capable of
+ let schema = Schema::new(vec![
+ make_field("i", DataType::Int64),
+ make_field("b", DataType::Boolean),
+ ])
+ .to_dfschema_ref()?;
+ let context = SimplifyContext::new(&props).with_schema(schema);
+ let simplifier = ExprSimplifier::new(context);
+
+ // basic arithmetic simplification
+ // i + 1 + 2 => a + 3
+ // (note this is not done if the expr is (col("i") + (lit(1) + lit(2))))
+ assert_eq!(
+ simplifier.simplify(col("i") + (lit(1) + lit(2)))?,
+ col("i") + lit(3)
+ );
+
+ // (i * 0) > 5 --> false (only if null)
+ assert_eq!(
+ simplifier.simplify((col("i") * lit(0)).gt(lit(5)))?,
+ lit(false)
+ );
+
+ // Logical simplification
+
+ // ((i > 5) AND FALSE) OR (i < 10) --> i < 10
+ assert_eq!(
+ simplifier
+ .simplify(col("i").gt(lit(5)).and(lit(false)).or(col("i").lt(lit(10))))?,
+ col("i").lt(lit(10))
+ );
+
+ Ok(())
+}
+
+fn make_field(name: &str, data_type: DataType) -> Field {
+ let nullable = false;
+ Field::new(name, data_type, nullable)
+}
+
+fn make_ts_field(name: &str) -> Field {
+ let tz = None;
+ make_field(name, DataType::Timestamp(TimeUnit::Nanosecond, tz))
+}
diff --git a/_downloads/97b381d1ef654142806e5e91c3788385/simple_udf.rs b/_downloads/97b381d1ef654142806e5e91c3788385/simple_udf.rs
new file mode 100644
index 0000000000..dba4385b8e
--- /dev/null
+++ b/_downloads/97b381d1ef654142806e5e91c3788385/simple_udf.rs
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion::{
+ arrow::{
+ array::{ArrayRef, Float32Array, Float64Array},
+ datatypes::DataType,
+ record_batch::RecordBatch,
+ },
+ logical_expr::Volatility,
+};
+
+use datafusion::prelude::*;
+use datafusion::{error::Result, physical_plan::functions::make_scalar_function};
+use datafusion_common::cast::as_float64_array;
+use std::sync::Arc;
+
+// create local execution context with an in-memory table
+fn create_context() -> Result<SessionContext> {
+ use datafusion::arrow::datatypes::{Field, Schema};
+ // define a schema.
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Float32, false),
+ Field::new("b", DataType::Float64, false),
+ ]));
+
+ // define data.
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Float32Array::from(vec![2.1, 3.1, 4.1, 5.1])),
+ Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])),
+ ],
+ )?;
+
+ // declare a new context. In spark API, this corresponds to a new spark SQLsession
+ let ctx = SessionContext::new();
+
+ // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+ ctx.register_batch("t", batch)?;
+ Ok(ctx)
+}
+
+/// In this example we will declare a single-type, single return type UDF that exponentiates f64, a^b
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = create_context()?;
+
+ // First, declare the actual implementation of the calculation
+ let pow = |args: &[ArrayRef]| {
+ // in DataFusion, all `args` and output are dynamically-typed arrays, which means that we need to:
+ // 1. cast the values to the type we want
+ // 2. perform the computation for every element in the array (using a loop or SIMD) and construct the result
+
+ // this is guaranteed by DataFusion based on the function's signature.
+ assert_eq!(args.len(), 2);
+
+ // 1. cast both arguments to f64. These casts MUST be aligned with the signature or this function panics!
+ let base = as_float64_array(&args[0]).expect("cast failed");
+ let exponent = as_float64_array(&args[1]).expect("cast failed");
+
+ // this is guaranteed by DataFusion. We place it just to make it obvious.
+ assert_eq!(exponent.len(), base.len());
+
+ // 2. perform the computation
+ let array = base
+ .iter()
+ .zip(exponent.iter())
+ .map(|(base, exponent)| {
+ match (base, exponent) {
+ // in arrow, any value can be null.
+ // Here we decide to make our UDF to return null when either base or exponent is null.
+ (Some(base), Some(exponent)) => Some(base.powf(exponent)),
+ _ => None,
+ }
+ })
+ .collect::<Float64Array>();
+
+ // `Ok` because no error occurred during the calculation (we should add one if exponent was [0, 1[ and the base < 0 because that panics!)
+ // `Arc` because arrays are immutable, thread-safe, trait objects.
+ Ok(Arc::new(array) as ArrayRef)
+ };
+ // the function above expects an `ArrayRef`, but DataFusion may pass a scalar to a UDF.
+ // thus, we use `make_scalar_function` to decorare the closure so that it can handle both Arrays and Scalar values.
+ let pow = make_scalar_function(pow);
+
+ // Next:
+ // * give it a name so that it shows nicely when the plan is printed
+ // * declare what input it expects
+ // * declare its return type
+ let pow = create_udf(
+ "pow",
+ // expects two f64
+ vec![DataType::Float64, DataType::Float64],
+ // returns f64
+ Arc::new(DataType::Float64),
+ Volatility::Immutable,
+ pow,
+ );
+
+ // at this point, we can use it or register it, depending on the use-case:
+ // * if the UDF is expected to be used throughout the program in different contexts,
+ // we can register it, and call it later:
+ ctx.register_udf(pow.clone()); // clone is only required in this example because we show both usages
+
+ // * if the UDF is expected to be used directly in the scope, `.call` it directly:
+ let expr = pow.call(vec![col("a"), col("b")]);
+
+ // get a DataFrame from the context
+ let df = ctx.table("t").await?;
+
+ // if we do not have `pow` in the scope and we registered it, we can get it from the registry
+ let pow = df.registry().udf("pow")?;
+ // equivalent to expr
+ let expr1 = pow.call(vec![col("a"), col("b")]);
+
+ // equivalent to `'SELECT pow(a, b), pow(a, b) AS pow1 FROM t'`
+ let df = df.select(vec![
+ expr,
+ // alias so that they have different column names
+ expr1.alias("pow1"),
+ ])?;
+
+ // note that "b" is f32, not f64. DataFusion coerces the types to match the UDF's signature.
+
+ // print the results
+ df.show().await?;
+
+ Ok(())
+}
diff --git a/_downloads/9d6db4688a18f2550d093295ce0f3231/simple_udwf.rs b/_downloads/9d6db4688a18f2550d093295ce0f3231/simple_udwf.rs
new file mode 100644
index 0000000000..15cfffe65f
--- /dev/null
+++ b/_downloads/9d6db4688a18f2550d093295ce0f3231/simple_udwf.rs
@@ -0,0 +1,211 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::{
+ array::{ArrayRef, AsArray, Float64Array},
+ datatypes::Float64Type,
+};
+use arrow_schema::DataType;
+use datafusion::datasource::file_format::options::CsvReadOptions;
+
+use datafusion::error::Result;
+use datafusion::prelude::*;
+use datafusion_common::{plan_err, DataFusionError, ScalarValue};
+use datafusion_expr::{
+ PartitionEvaluator, Signature, Volatility, WindowFrame, WindowUDF,
+};
+
+// create local execution context with `cars.csv` registered as a table named `cars`
+async fn create_context() -> Result<SessionContext> {
+ // declare a new context. In spark API, this corresponds to a new spark SQL session
+ let ctx = SessionContext::new();
+
+ // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+ println!("pwd: {}", std::env::current_dir().unwrap().display());
+ let csv_path = "datafusion/core/tests/data/cars.csv".to_string();
+ let read_options = CsvReadOptions::default().has_header(true);
+
+ ctx.register_csv("cars", &csv_path, read_options).await?;
+ Ok(ctx)
+}
+
+/// In this example we will declare a user defined window function that computes a moving average and then run it using SQL
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = create_context().await?;
+
+ // register the window function with DataFusion so we can call it
+ ctx.register_udwf(smooth_it());
+
+ // Use SQL to run the new window function
+ let df = ctx.sql("SELECT * from cars").await?;
+ // print the results
+ df.show().await?;
+
+ // Use SQL to run the new window function:
+ //
+ // `PARTITION BY car`:each distinct value of car (red, and green)
+ // should be treated as a separate partition (and will result in
+ // creating a new `PartitionEvaluator`)
+ //
+ // `ORDER BY time`: within each partition ('green' or 'red') the
+ // rows will be be ordered by the value in the `time` column
+ //
+ // `evaluate_inside_range` is invoked with a window defined by the
+ // SQL. In this case:
+ //
+ // The first invocation will be passed row 0, the first row in the
+ // partition.
+ //
+ // The second invocation will be passed rows 0 and 1, the first
+ // two rows in the partition.
+ //
+ // etc.
+ let df = ctx
+ .sql(
+ "SELECT \
+ car, \
+ speed, \
+ smooth_it(speed) OVER (PARTITION BY car ORDER BY time),\
+ time \
+ from cars \
+ ORDER BY \
+ car",
+ )
+ .await?;
+ // print the results
+ df.show().await?;
+
+ // this time, call the new widow function with an explicit
+ // window so evaluate will be invoked with each window.
+ //
+ // `ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING`: each invocation
+ // sees at most 3 rows: the row before, the current row, and the 1
+ // row afterward.
+ let df = ctx.sql(
+ "SELECT \
+ car, \
+ speed, \
+ smooth_it(speed) OVER (PARTITION BY car ORDER BY time ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING),\
+ time \
+ from cars \
+ ORDER BY \
+ car",
+ ).await?;
+ // print the results
+ df.show().await?;
+
+ // Now, run the function using the DataFrame API:
+ let window_expr = smooth_it().call(
+ vec![col("speed")], // smooth_it(speed)
+ vec![col("car")], // PARTITION BY car
+ vec![col("time").sort(true, true)], // ORDER BY time ASC
+ WindowFrame::new(false),
+ );
+ let df = ctx.table("cars").await?.window(vec![window_expr])?;
+
+ // print the results
+ df.show().await?;
+
+ Ok(())
+}
+
+fn smooth_it() -> WindowUDF {
+ WindowUDF {
+ name: String::from("smooth_it"),
+ // it will take 1 arguments -- the column to smooth
+ signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable),
+ return_type: Arc::new(return_type),
+ partition_evaluator_factory: Arc::new(make_partition_evaluator),
+ }
+}
+
+/// Compute the return type of the smooth_it window function given
+/// arguments of `arg_types`.
+fn return_type(arg_types: &[DataType]) -> Result<Arc<DataType>> {
+ if arg_types.len() != 1 {
+ return plan_err!(
+ "my_udwf expects 1 argument, got {}: {:?}",
+ arg_types.len(),
+ arg_types
+ );
+ }
+ Ok(Arc::new(arg_types[0].clone()))
+}
+
+/// Create a `PartitionEvalutor` to evaluate this function on a new
+/// partition.
+fn make_partition_evaluator() -> Result<Box<dyn PartitionEvaluator>> {
+ Ok(Box::new(MyPartitionEvaluator::new()))
+}
+
+/// This implements the lowest level evaluation for a window function
+///
+/// It handles calculating the value of the window function for each
+/// distinct values of `PARTITION BY` (each car type in our example)
+#[derive(Clone, Debug)]
+struct MyPartitionEvaluator {}
+
+impl MyPartitionEvaluator {
+ fn new() -> Self {
+ Self {}
+ }
+}
+
+/// Different evaluation methods are called depending on the various
+/// settings of WindowUDF. This example uses the simplest and most
+/// general, `evaluate`. See `PartitionEvaluator` for the other more
+/// advanced uses.
+impl PartitionEvaluator for MyPartitionEvaluator {
+ /// Tell DataFusion the window function varies based on the value
+ /// of the window frame.
+ fn uses_window_frame(&self) -> bool {
+ true
+ }
+
+ /// This function is called once per input row.
+ ///
+ /// `range`specifies which indexes of `values` should be
+ /// considered for the calculation.
+ ///
+ /// Note this is the SLOWEST, but simplest, way to evaluate a
+ /// window function. It is much faster to implement
+ /// evaluate_all or evaluate_all_with_rank, if possible
+ fn evaluate(
+ &mut self,
+ values: &[ArrayRef],
+ range: &std::ops::Range<usize>,
+ ) -> Result<ScalarValue> {
+ // Again, the input argument is an array of floating
+ // point numbers to calculate a moving average
+ let arr: &Float64Array = values[0].as_ref().as_primitive::<Float64Type>();
+
+ let range_len = range.end - range.start;
+
+ // our smoothing function will average all the values in the
+ let output = if range_len > 0 {
+ let sum: f64 = arr.values().iter().skip(range.start).take(range_len).sum();
+ Some(sum / range_len as f64)
+ } else {
+ None
+ };
+
+ Ok(ScalarValue::Float64(output))
+ }
+}
diff --git a/_downloads/ef95791a547ed7bb24fa06bdcae9272d/simple_udaf.rs b/_downloads/ef95791a547ed7bb24fa06bdcae9272d/simple_udaf.rs
new file mode 100644
index 0000000000..7aec9698d9
--- /dev/null
+++ b/_downloads/ef95791a547ed7bb24fa06bdcae9272d/simple_udaf.rs
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// In this example we will declare a single-type, single return type UDAF that computes the geometric mean.
+/// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_mean
+use datafusion::arrow::{
+ array::ArrayRef, array::Float32Array, datatypes::DataType, record_batch::RecordBatch,
+};
+use datafusion::{error::Result, physical_plan::Accumulator};
+use datafusion::{logical_expr::Volatility, prelude::*, scalar::ScalarValue};
+use datafusion_common::cast::as_float64_array;
+use datafusion_expr::create_udaf;
+use std::sync::Arc;
+
+// create local session context with an in-memory table
+fn create_context() -> Result<SessionContext> {
+ use datafusion::arrow::datatypes::{Field, Schema};
+ use datafusion::datasource::MemTable;
+ // define a schema.
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));
+
+ // define data in two partitions
+ let batch1 = RecordBatch::try_new(
+ schema.clone(),
+ vec![Arc::new(Float32Array::from(vec![2.0, 4.0, 8.0]))],
+ )?;
+ let batch2 = RecordBatch::try_new(
+ schema.clone(),
+ vec![Arc::new(Float32Array::from(vec![64.0]))],
+ )?;
+
+ // declare a new context. In spark API, this corresponds to a new spark SQLsession
+ let ctx = SessionContext::new();
+
+ // declare a table in memory. In spark API, this corresponds to createDataFrame(...).
+ let provider = MemTable::try_new(schema, vec![vec![batch1], vec![batch2]])?;
+ ctx.register_table("t", Arc::new(provider))?;
+ Ok(ctx)
+}
+
+/// A UDAF has state across multiple rows, and thus we require a `struct` with that state.
+#[derive(Debug)]
+struct GeometricMean {
+ n: u32,
+ prod: f64,
+}
+
+impl GeometricMean {
+ // how the struct is initialized
+ pub fn new() -> Self {
+ GeometricMean { n: 0, prod: 1.0 }
+ }
+}
+
+// UDAFs are built using the trait `Accumulator`, that offers DataFusion the necessary functions
+// to use them.
+impl Accumulator for GeometricMean {
+ // This function serializes our state to `ScalarValue`, which DataFusion uses
+ // to pass this state between execution stages.
+ // Note that this can be arbitrary data.
+ fn state(&self) -> Result<Vec<ScalarValue>> {
+ Ok(vec![
+ ScalarValue::from(self.prod),
+ ScalarValue::from(self.n),
+ ])
+ }
+
+ // DataFusion expects this function to return the final value of this aggregator.
+ // in this case, this is the formula of the geometric mean
+ fn evaluate(&self) -> Result<ScalarValue> {
+ let value = self.prod.powf(1.0 / self.n as f64);
+ Ok(ScalarValue::from(value))
+ }
+
+ // DataFusion calls this function to update the accumulator's state for a batch
+ // of inputs rows. In this case the product is updated with values from the first column
+ // and the count is updated based on the row count
+ fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
+ if values.is_empty() {
+ return Ok(());
+ }
+ let arr = &values[0];
+ (0..arr.len()).try_for_each(|index| {
+ let v = ScalarValue::try_from_array(arr, index)?;
+
+ if let ScalarValue::Float64(Some(value)) = v {
+ self.prod *= value;
+ self.n += 1;
+ } else {
+ unreachable!("")
+ }
+ Ok(())
+ })
+ }
+
+ // Optimization hint: this trait also supports `update_batch` and `merge_batch`,
+ // that can be used to perform these operations on arrays instead of single values.
+ fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
+ if states.is_empty() {
+ return Ok(());
+ }
+ let arr = &states[0];
+ (0..arr.len()).try_for_each(|index| {
+ let v = states
+ .iter()
+ .map(|array| ScalarValue::try_from_array(array, index))
+ .collect::<Result<Vec<_>>>()?;
+ if let (ScalarValue::Float64(Some(prod)), ScalarValue::UInt32(Some(n))) =
+ (&v[0], &v[1])
+ {
+ self.prod *= prod;
+ self.n += n;
+ } else {
+ unreachable!("")
+ }
+ Ok(())
+ })
+ }
+
+ fn size(&self) -> usize {
+ std::mem::size_of_val(self)
+ }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = create_context()?;
+
+ // here is where we define the UDAF. We also declare its signature:
+ let geometric_mean = create_udaf(
+ // the name; used to represent it in plan descriptions and in the registry, to use in SQL.
+ "geo_mean",
+ // the input type; DataFusion guarantees that the first entry of `values` in `update` has this type.
+ vec![DataType::Float64],
+ // the return type; DataFusion expects this to match the type returned by `evaluate`.
+ Arc::new(DataType::Float64),
+ Volatility::Immutable,
+ // This is the accumulator factory; DataFusion uses it to create new accumulators.
+ Arc::new(|_| Ok(Box::new(GeometricMean::new()))),
+ // This is the description of the state. `state()` must match the types here.
+ Arc::new(vec![DataType::Float64, DataType::UInt32]),
+ );
+
+ // get a DataFrame from the context
+ // this table has 1 column `a` f32 with values {2,4,8,64}, whose geometric mean is 8.0.
+ let df = ctx.table("t").await?;
+
+ // perform the aggregation
+ let df = df.aggregate(vec![], vec![geometric_mean.call(vec![col("a")])])?;
+
+ // note that "a" is f32, not f64. DataFusion coerces it to match the UDAF's signature.
+
+ // execute the query
+ let results = df.collect().await?;
+
+ // downcast the array to the expected type
+ let result = as_float64_array(results[0].column(0))?;
+
+ // verify that the calculation is correct
+ assert!((result.value(0) - 8.0).abs() < f64::EPSILON);
+ println!("The geometric mean of [2,4,8,64] is {}", result.value(0));
+
+ Ok(())
+}
diff --git a/_sources/library-user-guide/adding-udfs.md.txt b/_sources/library-user-guide/adding-udfs.md.txt
index 45d5afc1f0..d3f31bd45a 100644
--- a/_sources/library-user-guide/adding-udfs.md.txt
+++ b/_sources/library-user-guide/adding-udfs.md.txt
@@ -19,4 +19,106 @@
# Adding User Defined Functions: Scalar/Window/Aggregate
-Coming Soon
+User Defined Functions (UDFs) are functions that can be used in the context of DataFusion execution.
+
+This page covers how to add UDFs to DataFusion. In particular, it covers how to add Scalar, Window, and Aggregate UDFs.
+
+| UDF Type | Description | Example |
+| --------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
+| Scalar | A function that takes a row of data and returns a single value. | [simple_udf.rs](../../../datafusion-examples/examples/simple_udf.rs) |
+| Window | A function that takes a row of data and returns a single value, but also has access to the rows around it. | [simple_udwf.rs](../../../datafusion-examples/examples/simple_udwf.rs) |
+| Aggregate | A function that takes a group of rows and returns a single value. | [simple_udaf.rs](../../../datafusion-examples/examples/simple_udaf.rs) |
+
+First we'll talk about adding an Scalar UDF end-to-end, then we'll talk about the differences between the different types of UDFs.
+
+## Adding a Scalar UDF
+
+A Scalar UDF is a function that takes a row of data and returns a single value. For example, this function takes a single i64 and returns a single i64 with 1 added to it:
+
+```rust
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int64Array};
+use datafusion::common::Result;
+
+use datafusion::common::cast::as_int64_array;
+
+pub fn add_one(args: &[ArrayRef]) -> Result<ArrayRef> {
+ // Error handling omitted for brevity
+
+ let i64s = as_int64_array(&args[0])?;
+
+ let new_array = i64s
+ .iter()
+ .map(|array_elem| array_elem.map(|value| value + 1))
+ .collect::<Int64Array>();
+
+ Ok(Arc::new(new_array))
+}
+```
+
+For brevity, we'll skipped some error handling, but e.g. you may want to check that `args.len()` is the expected number of arguments.
+
+This "works" in isolation, i.e. if you have a slice of `ArrayRef`s, you can call `add_one` and it will return a new `ArrayRef` with 1 added to each value.
+
+```rust
+let input = vec![Some(1), None, Some(3)];
+let input = Arc::new(Int64Array::from(input)) as ArrayRef;
+
+let result = add_one(&[input]).unwrap();
+let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
+
+assert_eq!(result, &Int64Array::from(vec![Some(2), None, Some(4)]));
+```
+
+The challenge however is that DataFusion doesn't know about this function. We need to register it with DataFusion so that it can be used in the context of a query.
+
+### Registering a Scalar UDF
+
+To register a Scalar UDF, you need to wrap the function implementation in a `ScalarUDF` struct and then register it with the `SessionContext`. DataFusion provides the `create_udf` and `make_scalar_function` helper functions to make this easier.
+
+```rust
+let udf = create_udf(
+ "add_one",
+ vec![DataType::Int64],
+ Arc::new(DataType::Int64),
+ Volatility::Immutable,
+ make_scalar_function(add_one),
+);
+```
+
+A few things to note:
+
+- The first argument is the name of the function. This is the name that will be used in SQL queries.
+- The second argument is a vector of `DataType`s. This is the list of argument types that the function accepts. I.e. in this case, the function accepts a single `Int64` argument.
+- The third argument is the return type of the function. I.e. in this case, the function returns an `Int64`.
+- The fourth argument is the volatility of the function. In short, this is used to determine if the function's performance can be optimized in some situations. In this case, the function is `Immutable` because it always returns the same value for the same input. A random number generator would be `Volatile` because it returns a different value for the same input.
+- The fifth argument is the function implementation. This is the function that we defined above.
+
+That gives us a `ScalarUDF` that we can register with the `SessionContext`:
+
+```rust
+let mut ctx = SessionContext::new();
+
+ctx.register_udf(udf);
+```
+
+At this point, you can use the `add_one` function in your query:
+
+```rust
+let sql = "SELECT add_one(1)";
+
+let df = ctx.sql(&sql).await.unwrap();
+```
+
+## Adding a Window UDF
+
+Scalar UDFs are functions that take a row of data and return a single value. Window UDFs are similar, but they also have access to the rows around them. Access to the the proximal rows is helpful, but adds some complexity to the implementation.
+
+Body coming soon.
+
+## Adding an Aggregate UDF
+
+Aggregate UDFs are functions that take a group of rows and return a single value. These are akin to SQL's `SUM` or `COUNT` functions.
+
+Body coming soon.
diff --git a/_sources/library-user-guide/working-with-exprs.md.txt b/_sources/library-user-guide/working-with-exprs.md.txt
index b1a26cdfcb..507e984acb 100644
--- a/_sources/library-user-guide/working-with-exprs.md.txt
+++ b/_sources/library-user-guide/working-with-exprs.md.txt
@@ -19,4 +19,166 @@
# Working with Exprs
-Coming Soon
+<!-- https://github.com/apache/arrow-datafusion/issues/7304 -->
+
+`Expr` is short for "expression". It is a core abstraction in DataFusion for representing a computation, and follows the standard "expression tree" abstraction found in most compilers and databases.
+
+For example, the SQL expression `a + b` would be represented as an `Expr` with a `BinaryExpr` variant. A `BinaryExpr` has a left and right `Expr` and an operator.
+
+As another example, the SQL expression `a + b * c` would be represented as an `Expr` with a `BinaryExpr` variant. The left `Expr` would be `a` and the right `Expr` would be another `BinaryExpr` with a left `Expr` of `b` and a right `Expr` of `c`. As a classic expression tree, this would look like:
+
+```text
+ ┌────────────────────┐
+ │ BinaryExpr │
+ │ op: + │
+ └────────────────────┘
+ ▲ ▲
+ ┌───────┘ └────────────────┐
+ │ │
+┌────────────────────┐ ┌────────────────────┐
+│ Expr::Col │ │ BinaryExpr │
+│ col: a │ │ op: * │
+└────────────────────┘ └────────────────────┘
+ ▲ ▲
+ ┌────────┘ └─────────┐
+ │ │
+ ┌────────────────────┐ ┌────────────────────┐
+ │ Expr::Col │ │ Expr::Col │
+ │ col: b │ │ col: c │
+ └────────────────────┘ └────────────────────┘
+```
+
+As the writer of a library, you may want to use or create `Expr`s to represent computations that you want to perform. This guide will walk you through how to make your own scalar UDF as an `Expr` and how to rewrite `Expr`s to inline the simple UDF.
+
+There are also executable examples for working with `Expr`s:
+
+- [rewrite_expr.rs](../../../datafusion-examples/examples/catalog.rs)
+- [expr_api.rs](../../../datafusion-examples/examples/expr_api.rs)
+
+## A Scalar UDF Example
+
+We'll use a `ScalarUDF` expression as our example. This necessitates implementing an actual UDF, and for ease we'll use the same example from the [adding UDFs](./adding-udfs.md) guide.
+
+So assuming you've written that function, you can use it to create an `Expr`:
+
+```rust
+let add_one_udf = create_udf(
+ "add_one",
+ vec![DataType::Int64],
+ Arc::new(DataType::Int64),
+ Volatility::Immutable,
+ make_scalar_function(add_one), // <-- the function we wrote
+);
+
+// make the expr `add_one(5)`
+let expr = add_one_udf.call(vec![lit(5)]);
+
+// make the expr `add_one(my_column)`
+let expr = add_one_udf.call(vec![col("my_column")]);
+```
+
+If you'd like to learn more about `Expr`s, before we get into the details of creating and rewriting them, you can read the [expression user-guide](./../user-guide/expressions.md).
+
+## Rewriting Exprs
+
+Rewriting Expressions is the process of taking an `Expr` and transforming it into another `Expr`. This is useful for a number of reasons, including:
+
+- Simplifying `Expr`s to make them easier to evaluate
+- Optimizing `Expr`s to make them faster to evaluate
+- Converting `Expr`s to other forms, e.g. converting a `BinaryExpr` to a `CastExpr`
+
+In our example, we'll use rewriting to update our `add_one` UDF, to be rewritten as a `BinaryExpr` with a `Literal` of 1. We're effectively inlining the UDF.
+
+### Rewriting with `transform`
+
+To implement the inlining, we'll need to write a function that takes an `Expr` and returns a `Result<Expr>`. If the expression is _not_ to be rewritten `Transformed::No` is used to wrap the original `Expr`. If the expression _is_ to be rewritten, `Transformed::Yes` is used to wrap the new `Expr`.
+
+```rust
+fn rewrite_add_one(expr: Expr) -> Result<Expr> {
+ expr.transform(&|expr| {
+ Ok(match expr {
+ Expr::ScalarUDF(scalar_fun) if scalar_fun.fun.name == "add_one" => {
+ let input_arg = scalar_fun.args[0].clone();
+ let new_expression = input_arg + lit(1i64);
+
+ Transformed::Yes(new_expression)
+ }
+ _ => Transformed::No(expr),
+ })
+ })
+}
+```
+
+### Creating an `OptimizerRule`
+
+In DataFusion, an `OptimizerRule` is a trait that supports rewriting`Expr`s that appear in various parts of the `LogicalPlan`. It follows DataFusion's general mantra of trait implementations to drive behavior.
+
+We'll call our rule `AddOneInliner` and implement the `OptimizerRule` trait. The `OptimizerRule` trait has two methods:
+
+- `name` - returns the name of the rule
+- `try_optimize` - takes a `LogicalPlan` and returns an `Option<LogicalPlan>`. If the rule is able to optimize the plan, it returns `Some(LogicalPlan)` with the optimized plan. If the rule is not able to optimize the plan, it returns `None`.
+
+```rust
+struct AddOneInliner {}
+
+impl OptimizerRule for AddOneInliner {
+ fn name(&self) -> &str {
+ "add_one_inliner"
+ }
+
+ fn try_optimize(
+ &self,
+ plan: &LogicalPlan,
+ config: &dyn OptimizerConfig,
+ ) -> Result<Option<LogicalPlan>> {
+ // Map over the expressions and rewrite them
+ let new_expressions = plan
+ .expressions()
+ .into_iter()
+ .map(|expr| rewrite_add_one(expr))
+ .collect::<Result<Vec<_>>>()?;
+
+ let inputs = plan.inputs().into_iter().cloned().collect::<Vec<_>>();
+
+ let plan = plan.with_new_exprs(&new_expressions, &inputs);
+
+ plan.map(Some)
+ }
+}
+```
+
+Note the use of `rewrite_add_one` which is mapped over `plan.expressions()` to rewrite the expressions, then `plan.with_new_exprs` is used to create a new `LogicalPlan` with the rewritten expressions.
+
+We're almost there. Let's just test our rule works properly.
+
+## Testing the Rule
+
+Testing the rule is fairly simple, we can create a SessionState with our rule and then create a DataFrame and run a query. The logical plan will be optimized by our rule.
+
+```rust
+use datafusion::prelude::*;
+
+let rules = Arc::new(AddOneInliner {});
+let state = ctx.state().with_optimizer_rules(vec![rules]);
+
+let ctx = SessionContext::with_state(state);
+ctx.register_udf(add_one);
+
+let sql = "SELECT add_one(1) AS added_one";
+let plan = ctx.sql(sql).await?.logical_plan();
+
+println!("{:?}", plan);
+```
+
+This results in the following output:
+
+```text
+Projection: Int64(1) + Int64(1) AS added_one
+ EmptyRelation
+```
+
+I.e. the `add_one` UDF has been inlined into the projection.
+
+## Conclusion
+
+In this guide, we've seen how to create `Expr`s programmatically and how to rewrite them. This is useful for simplifying and optimizing `Expr`s. We've also seen how to test our rule to ensure it works properly.
diff --git a/library-user-guide/adding-udfs.html b/library-user-guide/adding-udfs.html
index 4dcc621dcf..d947c993d5 100644
--- a/library-user-guide/adding-udfs.html
+++ b/library-user-guide/adding-udfs.html
@@ -334,9 +334,36 @@
<div class="toc-item">
+<div class="tocsection onthispage pt-5 pb-3">
+ <i class="fas fa-list"></i> On this page
+</div>
<nav id="bd-toc-nav">
-
+ <ul class="visible nav section-nav flex-column">
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#adding-a-scalar-udf">
+ Adding a Scalar UDF
+ </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#registering-a-scalar-udf">
+ Registering a Scalar UDF
+ </a>
+ </li>
+ </ul>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#adding-a-window-udf">
+ Adding a Window UDF
+ </a>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#adding-an-aggregate-udf">
+ Adding an Aggregate UDF
+ </a>
+ </li>
+</ul>
+
</nav>
</div>
@@ -383,7 +410,111 @@
-->
<section id="adding-user-defined-functions-scalar-window-aggregate">
<h1>Adding User Defined Functions: Scalar/Window/Aggregate<a class="headerlink" href="#adding-user-defined-functions-scalar-window-aggregate" title="Link to this heading">¶</a></h1>
-<p>Coming Soon</p>
+<p>User Defined Functions (UDFs) are functions that can be used in the context of DataFusion execution.</p>
+<p>This page covers how to add UDFs to DataFusion. In particular, it covers how to add Scalar, Window, and Aggregate UDFs.</p>
+<table class="table">
+<thead>
+<tr class="row-odd"><th class="head"><p>UDF Type</p></th>
+<th class="head"><p>Description</p></th>
+<th class="head"><p>Example</p></th>
+</tr>
+</thead>
+<tbody>
+<tr class="row-even"><td><p>Scalar</p></td>
+<td><p>A function that takes a row of data and returns a single value.</p></td>
+<td><p><a class="reference download internal" download="" href="../_downloads/97b381d1ef654142806e5e91c3788385/simple_udf.rs"><span class="xref download myst">simple_udf.rs</span></a></p></td>
+</tr>
+<tr class="row-odd"><td><p>Window</p></td>
+<td><p>A function that takes a row of data and returns a single value, but also has access to the rows around it.</p></td>
+<td><p><a class="reference download internal" download="" href="../_downloads/9d6db4688a18f2550d093295ce0f3231/simple_udwf.rs"><span class="xref download myst">simple_udwf.rs</span></a></p></td>
+</tr>
+<tr class="row-even"><td><p>Aggregate</p></td>
+<td><p>A function that takes a group of rows and returns a single value.</p></td>
+<td><p><a class="reference download internal" download="" href="../_downloads/ef95791a547ed7bb24fa06bdcae9272d/simple_udaf.rs"><span class="xref download myst">simple_udaf.rs</span></a></p></td>
+</tr>
+</tbody>
+</table>
+<p>First we’ll talk about adding an Scalar UDF end-to-end, then we’ll talk about the differences between the different types of UDFs.</p>
+<section id="adding-a-scalar-udf">
+<h2>Adding a Scalar UDF<a class="headerlink" href="#adding-a-scalar-udf" title="Link to this heading">¶</a></h2>
+<p>A Scalar UDF is a function that takes a row of data and returns a single value. For example, this function takes a single i64 and returns a single i64 with 1 added to it:</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">std</span>::<span class="n">sync</span>::<span class="n">Arc</span><span class="p">;</span>
+
+<span class="k">use</span><span class="w"> </span><span class="n">arrow</span>::<span class="n">array</span>::<span class="p">{</span><span class="n">ArrayRef</span><span class="p">,</span><span class="w"> </span><span class="n">Int64Array</span><span class="p">};</span>
+<span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">common</span>::<span class="nb">Result</span><span class="p">;</span>
+
+<span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">common</span>::<span class="n">cast</span>::<span class="n">as_int64_array</span><span class="p">;</span>
+
+<span class="k">pub</span><span class="w"> </span><span class="k">fn</span> <span class="nf">add_one</span><span class="p">(</span><span class="n">args</span>: <span class="kp">&</span><span class="p">[</span><span class="n">ArrayRef</span><span class="p">])</span><span class="w"> </span>-> <span class="nb">Result</span><span class="o"><</span><span class="n">ArrayRef</span><span class="o">></span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// Error handling omitted for brevity</span>
+
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">i64s</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">as_int64_array</span><span class="p">(</span><span class="o">&</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">new_array</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">i64s</span>
+<span class="w"> </span><span class="p">.</span><span class="n">iter</span><span class="p">()</span>
+<span class="w"> </span><span class="p">.</span><span class="n">map</span><span class="p">(</span><span class="o">|</span><span class="n">array_elem</span><span class="o">|</span><span class="w"> </span><span class="n">array_elem</span><span class="p">.</span><span class="n">map</span><span class="p">(</span><span class="o">|</span><span class="n">value</span><span class="o">|</span><span class="w"> </span><span class="n">value</span><span class="w"> </span><span class="o">+</span>< [...]
+<span class="w"> </span><span class="p">.</span><span class="n">collect</span>::<span class="o"><</span><span class="n">Int64Array</span><span class="o">></span><span class="p">();</span>
+
+<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Arc</span>::<span class="n">new</span><span class="p">(</span><span class="n">new_array</span><span class="p">))</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>For brevity, we’ll skipped some error handling, but e.g. you may want to check that <code class="docutils literal notranslate"><span class="pre">args.len()</span></code> is the expected number of arguments.</p>
+<p>This “works” in isolation, i.e. if you have a slice of <code class="docutils literal notranslate"><span class="pre">ArrayRef</span></code>s, you can call <code class="docutils literal notranslate"><span class="pre">add_one</span></code> and it will return a new <code class="docutils literal notranslate"><span class="pre">ArrayRef</span></code> with 1 added to each value.</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">input</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="fm">vec!</span><span class="p">[</span><span class="nb">Some</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span><span class="w"> </span><span class="nb">None</span><span class="p">,</span><span class="w"> </span><span [...]
+<span class="kd">let</span><span class="w"> </span><span class="n">input</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Arc</span>::<span class="n">new</span><span class="p">(</span><span class="n">Int64Array</span>::<span class="n">from</span><span class="p">(</span><span class="n">input</span><span class="p">))</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="n">ArrayRef</span><span class="p">;</span>
+
+<span class="kd">let</span><span class="w"> </span><span class="n">result</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">add_one</span><span class="p">(</span><span class="o">&</span><span class="p">[</span><span class="n">input</span><span class="p">]).</span><span class="n">unwrap</span><span class="p">();</span>
+<span class="kd">let</span><span class="w"> </span><span class="n">result</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">result</span><span class="p">.</span><span class="n">as_any</span><span class="p">().</span><span class="n">downcast_ref</span>::<span class="o"><</span><span class="n">Int64Array</span><span class="o">></span><span class="p">().</span><span class="n">unwrap</span><span class="p">();</span>
+
+<span class="fm">assert_eq!</span><span class="p">(</span><span class="n">result</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">Int64Array</span>::<span class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span class="p">[</span><span class="nb">Some</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span><span class="w"> </span><span class="nb">None</span><span class="p">,</span><span class=" [...]
+</pre></div>
+</div>
+<p>The challenge however is that DataFusion doesn’t know about this function. We need to register it with DataFusion so that it can be used in the context of a query.</p>
+<section id="registering-a-scalar-udf">
+<h3>Registering a Scalar UDF<a class="headerlink" href="#registering-a-scalar-udf" title="Link to this heading">¶</a></h3>
+<p>To register a Scalar UDF, you need to wrap the function implementation in a <code class="docutils literal notranslate"><span class="pre">ScalarUDF</span></code> struct and then register it with the <code class="docutils literal notranslate"><span class="pre">SessionContext</span></code>. DataFusion provides the <code class="docutils literal notranslate"><span class="pre">create_udf</span></code> and <code class="docutils literal notranslate"><span class="pre">make_scalar_function</spa [...]
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">udf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">create_udf</span><span class="p">(</span>
+<span class="w"> </span><span class="s">"add_one"</span><span class="p">,</span>
+<span class="w"> </span><span class="fm">vec!</span><span class="p">[</span><span class="n">DataType</span>::<span class="n">Int64</span><span class="p">],</span>
+<span class="w"> </span><span class="n">Arc</span>::<span class="n">new</span><span class="p">(</span><span class="n">DataType</span>::<span class="n">Int64</span><span class="p">),</span>
+<span class="w"> </span><span class="n">Volatility</span>::<span class="n">Immutable</span><span class="p">,</span>
+<span class="w"> </span><span class="n">make_scalar_function</span><span class="p">(</span><span class="n">add_one</span><span class="p">),</span>
+<span class="p">);</span>
+</pre></div>
+</div>
+<p>A few things to note:</p>
+<ul class="simple">
+<li><p>The first argument is the name of the function. This is the name that will be used in SQL queries.</p></li>
+<li><p>The second argument is a vector of <code class="docutils literal notranslate"><span class="pre">DataType</span></code>s. This is the list of argument types that the function accepts. I.e. in this case, the function accepts a single <code class="docutils literal notranslate"><span class="pre">Int64</span></code> argument.</p></li>
+<li><p>The third argument is the return type of the function. I.e. in this case, the function returns an <code class="docutils literal notranslate"><span class="pre">Int64</span></code>.</p></li>
+<li><p>The fourth argument is the volatility of the function. In short, this is used to determine if the function’s performance can be optimized in some situations. In this case, the function is <code class="docutils literal notranslate"><span class="pre">Immutable</span></code> because it always returns the same value for the same input. A random number generator would be <code class="docutils literal notranslate"><span class="pre">Volatile</span></code> because it returns a different v [...]
+<li><p>The fifth argument is the function implementation. This is the function that we defined above.</p></li>
+</ul>
+<p>That gives us a <code class="docutils literal notranslate"><span class="pre">ScalarUDF</span></code> that we can register with the <code class="docutils literal notranslate"><span class="pre">SessionContext</span></code>:</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="k">mut</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">SessionContext</span>::<span class="n">new</span><span class="p">();</span>
+
+<span class="n">ctx</span><span class="p">.</span><span class="n">register_udf</span><span class="p">(</span><span class="n">udf</span><span class="p">);</span>
+</pre></div>
+</div>
+<p>At this point, you can use the <code class="docutils literal notranslate"><span class="pre">add_one</span></code> function in your query:</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">sql</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"SELECT add_one(1)"</span><span class="p">;</span>
+
+<span class="kd">let</span><span class="w"> </span><span class="n">df</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">sql</span><span class="p">(</span><span class="o">&</span><span class="n">sql</span><span class="p">).</span><span class="k">await</span><span class="p">.</span><span class="n">unwrap</span><span class="p">();</span>
+</pre></div>
+</div>
+</section>
+</section>
+<section id="adding-a-window-udf">
+<h2>Adding a Window UDF<a class="headerlink" href="#adding-a-window-udf" title="Link to this heading">¶</a></h2>
+<p>Scalar UDFs are functions that take a row of data and return a single value. Window UDFs are similar, but they also have access to the rows around them. Access to the the proximal rows is helpful, but adds some complexity to the implementation.</p>
+<p>Body coming soon.</p>
+</section>
+<section id="adding-an-aggregate-udf">
+<h2>Adding an Aggregate UDF<a class="headerlink" href="#adding-an-aggregate-udf" title="Link to this heading">¶</a></h2>
+<p>Aggregate UDFs are functions that take a group of rows and return a single value. These are akin to SQL’s <code class="docutils literal notranslate"><span class="pre">SUM</span></code> or <code class="docutils literal notranslate"><span class="pre">COUNT</span></code> functions.</p>
+<p>Body coming soon.</p>
+</section>
</section>
diff --git a/library-user-guide/working-with-exprs.html b/library-user-guide/working-with-exprs.html
index 927524b5da..a294d20dc8 100644
--- a/library-user-guide/working-with-exprs.html
+++ b/library-user-guide/working-with-exprs.html
@@ -334,9 +334,56 @@
<div class="toc-item">
+<div class="tocsection onthispage pt-5 pb-3">
+ <i class="fas fa-list"></i> On this page
+</div>
<nav id="bd-toc-nav">
-
+ <ul class="visible nav section-nav flex-column">
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#a-scalar-udf-example">
+ A Scalar UDF Example
+ </a>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#rewriting-exprs">
+ Rewriting Exprs
+ </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#rewriting-with-transform">
+ Rewriting with
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ transform
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#creating-an-optimizerrule">
+ Creating an
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ OptimizerRule
+ </span>
+ </code>
+ </a>
+ </li>
+ </ul>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#testing-the-rule">
+ Testing the Rule
+ </a>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#conclusion">
+ Conclusion
+ </a>
+ </li>
+</ul>
+
</nav>
</div>
@@ -383,7 +430,153 @@
-->
<section id="working-with-exprs">
<h1>Working with Exprs<a class="headerlink" href="#working-with-exprs" title="Link to this heading">¶</a></h1>
-<p>Coming Soon</p>
+<!-- https://github.com/apache/arrow-datafusion/issues/7304 -->
+<p><code class="docutils literal notranslate"><span class="pre">Expr</span></code> is short for “expression”. It is a core abstraction in DataFusion for representing a computation, and follows the standard “expression tree” abstraction found in most compilers and databases.</p>
+<p>For example, the SQL expression <code class="docutils literal notranslate"><span class="pre">a</span> <span class="pre">+</span> <span class="pre">b</span></code> would be represented as an <code class="docutils literal notranslate"><span class="pre">Expr</span></code> with a <code class="docutils literal notranslate"><span class="pre">BinaryExpr</span></code> variant. A <code class="docutils literal notranslate"><span class="pre">BinaryExpr</span></code> has a left and right <code cl [...]
+<p>As another example, the SQL expression <code class="docutils literal notranslate"><span class="pre">a</span> <span class="pre">+</span> <span class="pre">b</span> <span class="pre">*</span> <span class="pre">c</span></code> would be represented as an <code class="docutils literal notranslate"><span class="pre">Expr</span></code> with a <code class="docutils literal notranslate"><span class="pre">BinaryExpr</span></code> variant. The left <code class="docutils literal notranslate"><spa [...]
+<div class="highlight-text notranslate"><div class="highlight"><pre><span></span> ┌────────────────────┐
+ │ BinaryExpr │
+ │ op: + │
+ └────────────────────┘
+ ▲ ▲
+ ┌───────┘ └────────────────┐
+ │ │
+┌────────────────────┐ ┌────────────────────┐
+│ Expr::Col │ │ BinaryExpr │
+│ col: a │ │ op: * │
+└────────────────────┘ └────────────────────┘
+ ▲ ▲
+ ┌────────┘ └─────────┐
+ │ │
+ ┌────────────────────┐ ┌────────────────────┐
+ │ Expr::Col │ │ Expr::Col │
+ │ col: b │ │ col: c │
+ └────────────────────┘ └────────────────────┘
+</pre></div>
+</div>
+<p>As the writer of a library, you may want to use or create <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s to represent computations that you want to perform. This guide will walk you through how to make your own scalar UDF as an <code class="docutils literal notranslate"><span class="pre">Expr</span></code> and how to rewrite <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s to inline the simple UDF.</p>
+<p>There are also executable examples for working with <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s:</p>
+<ul class="simple">
+<li><p><a class="reference download internal" download="" href="../_downloads/9dc934d68aef6b44bd3235a7dbc9a8be/catalog.rs"><span class="xref download myst">rewrite_expr.rs</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../_downloads/1a0b5c48023f0bdbcb8dff8324195c3c/expr_api.rs"><span class="xref download myst">expr_api.rs</span></a></p></li>
+</ul>
+<section id="a-scalar-udf-example">
+<h2>A Scalar UDF Example<a class="headerlink" href="#a-scalar-udf-example" title="Link to this heading">¶</a></h2>
+<p>We’ll use a <code class="docutils literal notranslate"><span class="pre">ScalarUDF</span></code> expression as our example. This necessitates implementing an actual UDF, and for ease we’ll use the same example from the <a class="reference internal" href="adding-udfs.html"><span class="std std-doc">adding UDFs</span></a> guide.</p>
+<p>So assuming you’ve written that function, you can use it to create an <code class="docutils literal notranslate"><span class="pre">Expr</span></code>:</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">add_one_udf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">create_udf</span><span class="p">(</span>
+<span class="w"> </span><span class="s">"add_one"</span><span class="p">,</span>
+<span class="w"> </span><span class="fm">vec!</span><span class="p">[</span><span class="n">DataType</span>::<span class="n">Int64</span><span class="p">],</span>
+<span class="w"> </span><span class="n">Arc</span>::<span class="n">new</span><span class="p">(</span><span class="n">DataType</span>::<span class="n">Int64</span><span class="p">),</span>
+<span class="w"> </span><span class="n">Volatility</span>::<span class="n">Immutable</span><span class="p">,</span>
+<span class="w"> </span><span class="n">make_scalar_function</span><span class="p">(</span><span class="n">add_one</span><span class="p">),</span><span class="w"> </span><span class="c1">// <-- the function we wrote</span>
+<span class="p">);</span>
+
+<span class="c1">// make the expr `add_one(5)`</span>
+<span class="kd">let</span><span class="w"> </span><span class="n">expr</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">add_one_udf</span><span class="p">.</span><span class="n">call</span><span class="p">(</span><span class="fm">vec!</span><span class="p">[</span><span class="n">lit</span><span class="p">(</span><span class="mi">5</span><span class="p">)]);</span>
+
+<span class="c1">// make the expr `add_one(my_column)`</span>
+<span class="kd">let</span><span class="w"> </span><span class="n">expr</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">add_one_udf</span><span class="p">.</span><span class="n">call</span><span class="p">(</span><span class="fm">vec!</span><span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s">"my_column"</span><span class="p">)]);</span>
+</pre></div>
+</div>
+<p>If you’d like to learn more about <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s, before we get into the details of creating and rewriting them, you can read the <a class="reference internal" href="../user-guide/expressions.html"><span class="std std-doc">expression user-guide</span></a>.</p>
+</section>
+<section id="rewriting-exprs">
+<h2>Rewriting Exprs<a class="headerlink" href="#rewriting-exprs" title="Link to this heading">¶</a></h2>
+<p>Rewriting Expressions is the process of taking an <code class="docutils literal notranslate"><span class="pre">Expr</span></code> and transforming it into another <code class="docutils literal notranslate"><span class="pre">Expr</span></code>. This is useful for a number of reasons, including:</p>
+<ul class="simple">
+<li><p>Simplifying <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s to make them easier to evaluate</p></li>
+<li><p>Optimizing <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s to make them faster to evaluate</p></li>
+<li><p>Converting <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s to other forms, e.g. converting a <code class="docutils literal notranslate"><span class="pre">BinaryExpr</span></code> to a <code class="docutils literal notranslate"><span class="pre">CastExpr</span></code></p></li>
+</ul>
+<p>In our example, we’ll use rewriting to update our <code class="docutils literal notranslate"><span class="pre">add_one</span></code> UDF, to be rewritten as a <code class="docutils literal notranslate"><span class="pre">BinaryExpr</span></code> with a <code class="docutils literal notranslate"><span class="pre">Literal</span></code> of 1. We’re effectively inlining the UDF.</p>
+<section id="rewriting-with-transform">
+<h3>Rewriting with <code class="docutils literal notranslate"><span class="pre">transform</span></code><a class="headerlink" href="#rewriting-with-transform" title="Link to this heading">¶</a></h3>
+<p>To implement the inlining, we’ll need to write a function that takes an <code class="docutils literal notranslate"><span class="pre">Expr</span></code> and returns a <code class="docutils literal notranslate"><span class="pre">Result<Expr></span></code>. If the expression is <em>not</em> to be rewritten <code class="docutils literal notranslate"><span class="pre">Transformed::No</span></code> is used to wrap the original <code class="docutils literal notranslate"><span class="pr [...]
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">fn</span> <span class="nf">rewrite_add_one</span><span class="p">(</span><span class="n">expr</span>: <span class="nc">Expr</span><span class="p">)</span><span class="w"> </span>-> <span class="nb">Result</span><span class="o"><</span><span class="n">Expr</span><span class="o">></span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="n">expr</span><span class="p">.</span><span class="n">transform</span><span class="p">(</span><span class="o">&|</span><span class="n">expr</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="k">match</span><span class="w"> </span><span class="n">expr</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="n">Expr</span>::<span class="n">ScalarUDF</span><span class="p">(</span><span class="n">scalar_fun</span><span class="p">)</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="n">scalar_fun</span><span class="p">.</span><span class="n">fun</span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="s">"add_one" [...]
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">input_arg</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">scalar_fun</span><span class="p">.</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">].</span><span class="n">clone</span><span class="p">();</span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">new_expression</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">input_arg</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="k">i64</span><span class="p">);</span>
+
+<span class="w"> </span><span class="n">Transformed</span>::<span class="n">Yes</span><span class="p">(</span><span class="n">new_expression</span><span class="p">)</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="n">_</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="n">Transformed</span>::<span class="n">No</span><span class="p">(</span><span class="n">expr</span><span class="p">),</span>
+<span class="w"> </span><span class="p">})</span>
+<span class="w"> </span><span class="p">})</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+<section id="creating-an-optimizerrule">
+<h3>Creating an <code class="docutils literal notranslate"><span class="pre">OptimizerRule</span></code><a class="headerlink" href="#creating-an-optimizerrule" title="Link to this heading">¶</a></h3>
+<p>In DataFusion, an <code class="docutils literal notranslate"><span class="pre">OptimizerRule</span></code> is a trait that supports rewriting<code class="docutils literal notranslate"><span class="pre">Expr</span></code>s that appear in various parts of the <code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code>. It follows DataFusion’s general mantra of trait implementations to drive behavior.</p>
+<p>We’ll call our rule <code class="docutils literal notranslate"><span class="pre">AddOneInliner</span></code> and implement the <code class="docutils literal notranslate"><span class="pre">OptimizerRule</span></code> trait. The <code class="docutils literal notranslate"><span class="pre">OptimizerRule</span></code> trait has two methods:</p>
+<ul class="simple">
+<li><p><code class="docutils literal notranslate"><span class="pre">name</span></code> - returns the name of the rule</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">try_optimize</span></code> - takes a <code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code> and returns an <code class="docutils literal notranslate"><span class="pre">Option<LogicalPlan></span></code>. If the rule is able to optimize the plan, it returns <code class="docutils literal notranslate"><span class="pre">Some(LogicalPlan)</span></code> with the optimized plan. If the rule [...]
+</ul>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">struct</span> <span class="nc">AddOneInliner</span><span class="w"> </span><span class="p">{}</span>
+
+<span class="k">impl</span><span class="w"> </span><span class="n">OptimizerRule</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">AddOneInliner</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">fn</span> <span class="nf">name</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span>-> <span class="kp">&</span><span class="kt">str</span> <span class="p">{</span>
+<span class="w"> </span><span class="s">"add_one_inliner"</span>
+<span class="w"> </span><span class="p">}</span>
+
+<span class="w"> </span><span class="k">fn</span> <span class="nf">try_optimize</span><span class="p">(</span>
+<span class="w"> </span><span class="o">&</span><span class="bp">self</span><span class="p">,</span>
+<span class="w"> </span><span class="n">plan</span>: <span class="kp">&</span><span class="nc">LogicalPlan</span><span class="p">,</span>
+<span class="w"> </span><span class="n">config</span>: <span class="kp">&</span><span class="nc">dyn</span><span class="w"> </span><span class="n">OptimizerConfig</span><span class="p">,</span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span>-> <span class="nb">Result</span><span class="o"><</span><span class="nb">Option</span><span class="o"><</span><span class="n">LogicalPlan</span><span class="o">>></span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// Map over the expressions and rewrite them</span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">new_expressions</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">plan</span>
+<span class="w"> </span><span class="p">.</span><span class="n">expressions</span><span class="p">()</span>
+<span class="w"> </span><span class="p">.</span><span class="n">into_iter</span><span class="p">()</span>
+<span class="w"> </span><span class="p">.</span><span class="n">map</span><span class="p">(</span><span class="o">|</span><span class="n">expr</span><span class="o">|</span><span class="w"> </span><span class="n">rewrite_add_one</span><span class="p">(</span><span class="n">expr</span><span class="p">))</span>
+<span class="w"> </span><span class="p">.</span><span class="n">collect</span>::<span class="o"><</span><span class="nb">Result</span><span class="o"><</span><span class="nb">Vec</span><span class="o"><</span><span class="n">_</span><span class="o">>>></span><span class="p">()</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">inputs</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">plan</span><span class="p">.</span><span class="n">inputs</span><span class="p">().</span><span class="n">into_iter</span><span class="p">().</span><span class="n">cloned</span><span class="p">().</span><span class="n">collect</span>::<span class="o"><</span><span class="nb">Vec</span><spa [...]
+
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">plan</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">plan</span><span class="p">.</span><span class="n">with_new_exprs</span><span class="p">(</span><span class="o">&</span><span class="n">new_expressions</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">inputs</span><span class="p">);</span>
+
+<span class="w"> </span><span class="n">plan</span><span class="p">.</span><span class="n">map</span><span class="p">(</span><span class="nb">Some</span><span class="p">)</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>Note the use of <code class="docutils literal notranslate"><span class="pre">rewrite_add_one</span></code> which is mapped over <code class="docutils literal notranslate"><span class="pre">plan.expressions()</span></code> to rewrite the expressions, then <code class="docutils literal notranslate"><span class="pre">plan.with_new_exprs</span></code> is used to create a new <code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code> with the rewritten expressions.</p>
+<p>We’re almost there. Let’s just test our rule works properly.</p>
+</section>
+</section>
+<section id="testing-the-rule">
+<h2>Testing the Rule<a class="headerlink" href="#testing-the-rule" title="Link to this heading">¶</a></h2>
+<p>Testing the rule is fairly simple, we can create a SessionState with our rule and then create a DataFrame and run a query. The logical plan will be optimized by our rule.</p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">prelude</span>::<span class="o">*</span><span class="p">;</span>
+
+<span class="kd">let</span><span class="w"> </span><span class="n">rules</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Arc</span>::<span class="n">new</span><span class="p">(</span><span class="n">AddOneInliner</span><span class="w"> </span><span class="p">{});</span>
+<span class="kd">let</span><span class="w"> </span><span class="n">state</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">state</span><span class="p">().</span><span class="n">with_optimizer_rules</span><span class="p">(</span><span class="fm">vec!</span><span class="p">[</span><span class="n">rules</span><span class="p">]);</span>
+
+<span class="kd">let</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">SessionContext</span>::<span class="n">with_state</span><span class="p">(</span><span class="n">state</span><span class="p">);</span>
+<span class="n">ctx</span><span class="p">.</span><span class="n">register_udf</span><span class="p">(</span><span class="n">add_one</span><span class="p">);</span>
+
+<span class="kd">let</span><span class="w"> </span><span class="n">sql</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"SELECT add_one(1) AS added_one"</span><span class="p">;</span>
+<span class="kd">let</span><span class="w"> </span><span class="n">plan</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">sql</span><span class="p">(</span><span class="n">sql</span><span class="p">).</span><span class="k">await</span><span class="o">?</span><span class="p">.</span><span class="n">logical_plan</span><span class="p">();</span>
+
+<span class="fm">println!</span><span class="p">(</span><span class="s">"{:?}"</span><span class="p">,</span><span class="w"> </span><span class="n">plan</span><span class="p">);</span>
+</pre></div>
+</div>
+<p>This results in the following output:</p>
+<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>Projection: Int64(1) + Int64(1) AS added_one
+ EmptyRelation
+</pre></div>
+</div>
+<p>I.e. the <code class="docutils literal notranslate"><span class="pre">add_one</span></code> UDF has been inlined into the projection.</p>
+</section>
+<section id="conclusion">
+<h2>Conclusion<a class="headerlink" href="#conclusion" title="Link to this heading">¶</a></h2>
+<p>In this guide, we’ve seen how to create <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s programmatically and how to rewrite them. This is useful for simplifying and optimizing <code class="docutils literal notranslate"><span class="pre">Expr</span></code>s. We’ve also seen how to test our rule to ensure it works properly.</p>
+</section>
</section>
diff --git a/searchindex.js b/searchindex.js
index 42cf8231b5..0bbe57460c 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["contributor-guide/architecture", "contributor-guide/communication", "contributor-guide/index", "contributor-guide/quarterly_roadmap", "contributor-guide/roadmap", "contributor-guide/specification/index", "contributor-guide/specification/invariants", "contributor-guide/specification/output-field-name-semantic", "index", "library-user-guide/adding-udfs", "library-user-guide/building-logical-plans", "library-user-guide/catalogs", "library-user-guide/custom-tab [...]
\ No newline at end of file
+Search.setIndex({"docnames": ["contributor-guide/architecture", "contributor-guide/communication", "contributor-guide/index", "contributor-guide/quarterly_roadmap", "contributor-guide/roadmap", "contributor-guide/specification/index", "contributor-guide/specification/invariants", "contributor-guide/specification/output-field-name-semantic", "index", "library-user-guide/adding-udfs", "library-user-guide/building-logical-plans", "library-user-guide/catalogs", "library-user-guide/custom-tab [...]
\ No newline at end of file