You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2024/02/02 14:58:43 UTC
(arrow-datafusion) branch alamb/array_crate created (now 8d1c0a1395)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a change to branch alamb/array_crate
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
at 8d1c0a1395 improve docs
This branch includes the following new commits:
new d013041f23 Add `datafusion-functions-array` crate
new 2d6512aa56 Fix columnar value to array
new 8d1c0a1395 improve docs
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(arrow-datafusion) 03/03: improve docs
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch alamb/array_crate
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 8d1c0a1395b4a4a552c6feb9419b53dd2f57b7a8
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Feb 2 09:58:26 2024 -0500
improve docs
---
datafusion/expr/src/columnar_value.rs | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/datafusion/expr/src/columnar_value.rs b/datafusion/expr/src/columnar_value.rs
index 5f2481ebc6..378ef52f01 100644
--- a/datafusion/expr/src/columnar_value.rs
+++ b/datafusion/expr/src/columnar_value.rs
@@ -78,9 +78,18 @@ impl ColumnarValue {
/// A helper function used to infer the length of arguments of Scalar
/// functions and convert [`ColumnarValue`]s to [`ArrayRef`]s with the
- /// inferred length. Note that this function only works for functions that
- /// accept either all scalar arguments or all arguments are arrays
- /// with same length. Otherwise, it will return an error.
+ /// inferred length.
+ ///
+ /// # Note on Performance
+ ///
+ /// This function expands any `ScalalarValues` to arrays of the same length.
+ /// This allows a single function implementaiton in terms of arrays, but it
+ /// can be inefficient. It is recommended to provide specialized
+ /// implementations for scalar values if performance is a concern.
+ ///
+ /// # Errors
+ ///
+ /// If there are multiple array arguments that have different lengths
pub fn values_to_arrays(args: &[ColumnarValue]) -> Result<Vec<ArrayRef>> {
if args.is_empty() {
return Ok(vec![]);
(arrow-datafusion) 02/03: Fix columnar value to array
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch alamb/array_crate
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 2d6512aa567ef1f0b9dbf7db55a704684858298c
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Feb 2 09:49:41 2024 -0500
Fix columnar value to array
---
datafusion/core/src/execution/context/mod.rs | 1 -
datafusion/expr/src/columnar_value.rs | 40 +++++++++++-----------------
2 files changed, 16 insertions(+), 25 deletions(-)
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
index abeffa6354..3661f6973c 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -1353,7 +1353,6 @@ impl SessionState {
datafusion_functions_array::register_all(&mut new_self)
.expect("can not register array expressions");
-
new_self
}
/// Returns new [`SessionState`] using the provided
diff --git a/datafusion/expr/src/columnar_value.rs b/datafusion/expr/src/columnar_value.rs
index 95369c2242..5f2481ebc6 100644
--- a/datafusion/expr/src/columnar_value.rs
+++ b/datafusion/expr/src/columnar_value.rs
@@ -20,7 +20,7 @@
use arrow::array::ArrayRef;
use arrow::array::NullArray;
use arrow::datatypes::DataType;
-use datafusion_common::{DataFusionError, Result, ScalarValue};
+use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue};
use std::sync::Arc;
/// Represents the result of evaluating an expression: either a single
@@ -86,33 +86,25 @@ impl ColumnarValue {
return Ok(vec![]);
}
- let len = args
- .iter()
- .fold(Option::<usize>::None, |acc, arg| match arg {
- ColumnarValue::Scalar(_) if acc.is_none() => Some(1),
- ColumnarValue::Scalar(_) => {
- if let Some(1) = acc {
- acc
+ let mut array_len = None;
+ for arg in args {
+ array_len = match (arg, array_len) {
+ (ColumnarValue::Array(a), None) => Some(a.len()),
+ (ColumnarValue::Array(a), Some(array_len)) => {
+ if array_len == a.len() {
+ Some(array_len)
} else {
- None
+ return internal_err!(
+ "Arguments has mixed length. Expected length: {array_len}, found length: {}",a.len()
+ );
}
}
- ColumnarValue::Array(a) => {
- if let Some(l) = acc {
- if l == a.len() {
- acc
- } else {
- None
- }
- } else {
- Some(a.len())
- }
- }
- });
+ (ColumnarValue::Scalar(_), array_len) => array_len,
+ }
+ }
- let inferred_length = len.ok_or(DataFusionError::Internal(
- "Arguments has mixed length".to_string(),
- ))?;
+ // If array_len is none, it means there are only scalars, so make a 1 element array
+ let inferred_length = array_len.unwrap_or(1);
let args = args
.iter()
(arrow-datafusion) 01/03: Add `datafusion-functions-array` crate
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch alamb/array_crate
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit d013041f2318a0e99969d82b5a7a7ca365d73a49
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Feb 2 06:40:27 2024 -0500
Add `datafusion-functions-array` crate
---
.github/workflows/rust.yml | 3 +
Cargo.toml | 3 +-
datafusion-examples/examples/simple_udf.rs | 3 +-
datafusion/core/Cargo.toml | 6 +-
datafusion/core/src/execution/context/mod.rs | 6 +
datafusion/expr/src/built_in_function.rs | 13 --
datafusion/expr/src/columnar_value.rs | 48 +++-
datafusion/expr/src/expr_fn.rs | 7 -
datafusion/functions-array/Cargo.toml | 45 ++++
datafusion/functions-array/README.md | 27 +++
datafusion/functions-array/src/kernels.rs | 254 +++++++++++++++++++++
datafusion/functions-array/src/lib.rs | 56 +++++
datafusion/functions-array/src/macros.rs | 77 +++++++
datafusion/functions-array/src/udf.rs | 85 +++++++
.../src/simplify_expressions/expr_simplifier.rs | 3 +-
datafusion/physical-expr/src/array_expressions.rs | 214 -----------------
datafusion/physical-expr/src/functions.rs | 47 +---
datafusion/proto/proto/datafusion.proto | 2 +-
datafusion/proto/src/generated/pbjson.rs | 3 -
datafusion/proto/src/generated/prost.rs | 4 +-
datafusion/proto/src/logical_plan/from_proto.rs | 14 +-
datafusion/proto/src/logical_plan/to_proto.rs | 1 -
22 files changed, 616 insertions(+), 305 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 501b05c25d..6fb7f3e340 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -82,6 +82,9 @@ jobs:
- name: Check function packages (encoding_expressions)
run: cargo check --no-default-features --features=encoding_expressions -p datafusion
+ - name: Check function packages (array_expressions)
+ run: cargo check --no-default-features --features=array_expressions -p datafusion
+
- name: Check Cargo.lock for datafusion-cli
run: |
# If this test fails, try running `cargo update` in the `datafusion-cli` directory
diff --git a/Cargo.toml b/Cargo.toml
index d56d37ad2b..35ea6365a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,7 @@
[workspace]
exclude = ["datafusion-cli"]
-members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/functions", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/physical-plan", "datafusion/proto", "datafusion/proto/gen", "datafusion/sql", "datafusion/sqllogictest", "datafusion/substrait", "datafusion/wasmtest", "datafusion-examples", "docs", "test-utils", "benchmarks",
+members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/functions", "datafusion/functions-array", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/physical-plan", "datafusion/proto", "datafusion/proto/gen", "datafusion/sql", "datafusion/sqllogictest", "datafusion/substrait", "datafusion/wasmtest", "datafusion-examples", "docs", "test-utils", "benchmarks",
]
resolver = "2"
@@ -50,6 +50,7 @@ datafusion-common = { path = "datafusion/common", version = "35.0.0" }
datafusion-execution = { path = "datafusion/execution", version = "35.0.0" }
datafusion-expr = { path = "datafusion/expr", version = "35.0.0" }
datafusion-functions = { path = "datafusion/functions", version = "35.0.0" }
+datafusion-functions-array = { path = "datafusion/functions-array", version = "35.0.0" }
datafusion-optimizer = { path = "datafusion/optimizer", version = "35.0.0" }
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "35.0.0" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "35.0.0" }
diff --git a/datafusion-examples/examples/simple_udf.rs b/datafusion-examples/examples/simple_udf.rs
index dda6ba62e0..9de5f1ddeb 100644
--- a/datafusion-examples/examples/simple_udf.rs
+++ b/datafusion-examples/examples/simple_udf.rs
@@ -28,7 +28,6 @@ use datafusion::error::Result;
use datafusion::prelude::*;
use datafusion_common::cast::as_float64_array;
use datafusion_expr::ColumnarValue;
-use datafusion_physical_expr::functions::columnar_values_to_array;
use std::sync::Arc;
/// create local execution context with an in-memory table:
@@ -71,7 +70,7 @@ async fn main() -> Result<()> {
// this is guaranteed by DataFusion based on the function's signature.
assert_eq!(args.len(), 2);
- let args = columnar_values_to_array(args)?;
+ let args = ColumnarValue::values_to_arrays(args)?;
// 1. cast both arguments to f64. These casts MUST be aligned with the signature or this function panics!
let base = as_float64_array(&args[0]).expect("cast failed");
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index f9a4c54b7d..3e28291a34 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -35,11 +35,12 @@ path = "src/lib.rs"
[features]
# Used to enable the avro format
+array_expressions = ["datafusion-functions-array"]
avro = ["apache-avro", "num-traits", "datafusion-common/avro"]
backtrace = ["datafusion-common/backtrace"]
compression = ["xz2", "bzip2", "flate2", "zstd", "async-compression", "tokio-util"]
crypto_expressions = ["datafusion-physical-expr/crypto_expressions", "datafusion-optimizer/crypto_expressions"]
-default = ["crypto_expressions", "encoding_expressions", "regex_expressions", "unicode_expressions", "compression", "parquet"]
+default = ["array_expressions", "crypto_expressions", "encoding_expressions", "regex_expressions", "unicode_expressions", "compression", "parquet"]
encoding_expressions = ["datafusion-functions/encoding_expressions"]
# Used for testing ONLY: causes all values to hash to the same value (test for collisions)
force_hash_collisions = []
@@ -65,7 +66,8 @@ dashmap = { workspace = true }
datafusion-common = { path = "../common", version = "35.0.0", features = ["object_store"], default-features = false }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
-datafusion-functions = { path = "../functions", version = "35.0.0" }
+datafusion-functions = {workspace = true }
+datafusion-functions-array = { workspace=true, optional = true }
datafusion-optimizer = { path = "../optimizer", version = "35.0.0", default-features = false }
datafusion-physical-expr = { path = "../physical-expr", version = "35.0.0", default-features = false }
datafusion-physical-plan = { workspace = true }
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
index 08a03f3c69..abeffa6354 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -1348,6 +1348,12 @@ impl SessionState {
datafusion_functions::register_all(&mut new_self)
.expect("can not register built in functions");
+ // register crate of array expressions (if enabled)
+ #[cfg(feature = "array_expressions")]
+ datafusion_functions_array::register_all(&mut new_self)
+ .expect("can not register array expressions");
+
+
new_self
}
/// Returns new [`SessionState`] using the provided
diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs
index b7bb17c86b..dc482b4740 100644
--- a/datafusion/expr/src/built_in_function.rs
+++ b/datafusion/expr/src/built_in_function.rs
@@ -175,8 +175,6 @@ pub enum BuiltinScalarFunction {
ArrayReverse,
/// array_slice
ArraySlice,
- /// array_to_string
- ArrayToString,
/// array_intersect
ArrayIntersect,
/// array_union
@@ -432,7 +430,6 @@ impl BuiltinScalarFunction {
BuiltinScalarFunction::ArrayReverse => Volatility::Immutable,
BuiltinScalarFunction::Flatten => Volatility::Immutable,
BuiltinScalarFunction::ArraySlice => Volatility::Immutable,
- BuiltinScalarFunction::ArrayToString => Volatility::Immutable,
BuiltinScalarFunction::ArrayIntersect => Volatility::Immutable,
BuiltinScalarFunction::ArrayUnion => Volatility::Immutable,
BuiltinScalarFunction::ArrayResize => Volatility::Immutable,
@@ -628,7 +625,6 @@ impl BuiltinScalarFunction {
BuiltinScalarFunction::ArrayReverse => Ok(input_expr_types[0].clone()),
BuiltinScalarFunction::ArraySlice => Ok(input_expr_types[0].clone()),
BuiltinScalarFunction::ArrayResize => Ok(input_expr_types[0].clone()),
- BuiltinScalarFunction::ArrayToString => Ok(Utf8),
BuiltinScalarFunction::ArrayIntersect => {
match (input_expr_types[0].clone(), input_expr_types[1].clone()) {
(DataType::Null, DataType::Null) | (DataType::Null, _) => {
@@ -970,9 +966,6 @@ impl BuiltinScalarFunction {
Signature::variadic_any(self.volatility())
}
- BuiltinScalarFunction::ArrayToString => {
- Signature::variadic_any(self.volatility())
- }
BuiltinScalarFunction::ArrayIntersect => Signature::any(2, self.volatility()),
BuiltinScalarFunction::ArrayUnion => Signature::any(2, self.volatility()),
BuiltinScalarFunction::Cardinality => Signature::any(1, self.volatility()),
@@ -1574,12 +1567,6 @@ impl BuiltinScalarFunction {
}
BuiltinScalarFunction::ArrayReverse => &["array_reverse", "list_reverse"],
BuiltinScalarFunction::ArraySlice => &["array_slice", "list_slice"],
- BuiltinScalarFunction::ArrayToString => &[
- "array_to_string",
- "list_to_string",
- "array_join",
- "list_join",
- ],
BuiltinScalarFunction::ArrayUnion => &["array_union", "list_union"],
BuiltinScalarFunction::Cardinality => &["cardinality"],
BuiltinScalarFunction::ArrayResize => &["array_resize", "list_resize"],
diff --git a/datafusion/expr/src/columnar_value.rs b/datafusion/expr/src/columnar_value.rs
index 58c534b50a..95369c2242 100644
--- a/datafusion/expr/src/columnar_value.rs
+++ b/datafusion/expr/src/columnar_value.rs
@@ -20,7 +20,7 @@
use arrow::array::ArrayRef;
use arrow::array::NullArray;
use arrow::datatypes::DataType;
-use datafusion_common::{Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue};
use std::sync::Arc;
/// Represents the result of evaluating an expression: either a single
@@ -75,4 +75,50 @@ impl ColumnarValue {
pub fn create_null_array(num_rows: usize) -> Self {
ColumnarValue::Array(Arc::new(NullArray::new(num_rows)))
}
+
+ /// A helper function used to infer the length of arguments of Scalar
+ /// functions and convert [`ColumnarValue`]s to [`ArrayRef`]s with the
+ /// inferred length. Note that this function only works for functions that
+ /// accept either all scalar arguments or all arguments are arrays
+ /// with same length. Otherwise, it will return an error.
+ pub fn values_to_arrays(args: &[ColumnarValue]) -> Result<Vec<ArrayRef>> {
+ if args.is_empty() {
+ return Ok(vec![]);
+ }
+
+ let len = args
+ .iter()
+ .fold(Option::<usize>::None, |acc, arg| match arg {
+ ColumnarValue::Scalar(_) if acc.is_none() => Some(1),
+ ColumnarValue::Scalar(_) => {
+ if let Some(1) = acc {
+ acc
+ } else {
+ None
+ }
+ }
+ ColumnarValue::Array(a) => {
+ if let Some(l) = acc {
+ if l == a.len() {
+ acc
+ } else {
+ None
+ }
+ } else {
+ Some(a.len())
+ }
+ }
+ });
+
+ let inferred_length = len.ok_or(DataFusionError::Internal(
+ "Arguments has mixed length".to_string(),
+ ))?;
+
+ let args = args
+ .iter()
+ .map(|arg| arg.clone().into_array(inferred_length))
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(args)
+ }
}
diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs
index 877066aabf..674d55e8ee 100644
--- a/datafusion/expr/src/expr_fn.rs
+++ b/datafusion/expr/src/expr_fn.rs
@@ -740,12 +740,6 @@ scalar_expr!(
array begin end stride,
"returns a slice of the array."
);
-scalar_expr!(
- ArrayToString,
- array_to_string,
- array delimiter,
- "converts each element to its text representation."
-);
scalar_expr!(ArrayUnion, array_union, array1 array2, "returns an array of the elements in the union of array1 and array2 without duplicates.");
scalar_expr!(
@@ -1447,7 +1441,6 @@ mod test {
test_scalar_expr!(ArrayReplace, array_replace, array, from, to);
test_scalar_expr!(ArrayReplaceN, array_replace_n, array, from, to, max);
test_scalar_expr!(ArrayReplaceAll, array_replace_all, array, from, to);
- test_scalar_expr!(ArrayToString, array_to_string, array, delimiter);
test_unary_scalar_expr!(Cardinality, cardinality);
test_nary_scalar_expr!(MakeArray, array, input);
diff --git a/datafusion/functions-array/Cargo.toml b/datafusion/functions-array/Cargo.toml
new file mode 100644
index 0000000000..9cf769bf29
--- /dev/null
+++ b/datafusion/functions-array/Cargo.toml
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-functions-array"
+description = "Array Function packages for the DataFusion query engine"
+keywords = ["datafusion", "logical", "plan", "expressions"]
+readme = "README.md"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = { workspace = true }
+
+[features]
+
+[lib]
+name = "datafusion_functions_array"
+path = "src/lib.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+arrow = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-execution = { workspace = true }
+datafusion-expr = { workspace = true }
+log = "0.4.20"
+paste = "1.0.14"
diff --git a/datafusion/functions-array/README.md b/datafusion/functions-array/README.md
new file mode 100644
index 0000000000..032b65fae2
--- /dev/null
+++ b/datafusion/functions-array/README.md
@@ -0,0 +1,27 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# DataFusion Function Library
+
+[DataFusion][df] is an extensible query execution framework, written in Rust, that uses Apache Arrow as its in-memory format.
+
+This crate contains functions for working with arrays, such as `array_append` that work with
+`ListArray`, `LargeListArray` and `FixedListArray` types from the `arrow` crate.
+
+[df]: https://crates.io/crates/datafusion
diff --git a/datafusion/functions-array/src/kernels.rs b/datafusion/functions-array/src/kernels.rs
new file mode 100644
index 0000000000..1b96e01d8b
--- /dev/null
+++ b/datafusion/functions-array/src/kernels.rs
@@ -0,0 +1,254 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! implementation kernels for array functions
+
+use arrow::array::{
+ Array, ArrayRef, BooleanArray, Float32Array, Float64Array, GenericListArray,
+ Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, OffsetSizeTrait,
+ StringArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+};
+use arrow::datatypes::DataType;
+use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};
+use datafusion_common::{exec_err, DataFusionError};
+use std::any::type_name;
+use std::sync::Arc;
+
+macro_rules! downcast_arg {
+ ($ARG:expr, $ARRAY_TYPE:ident) => {{
+ $ARG.as_any().downcast_ref::<$ARRAY_TYPE>().ok_or_else(|| {
+ DataFusionError::Internal(format!(
+ "could not cast to {}",
+ type_name::<$ARRAY_TYPE>()
+ ))
+ })?
+ }};
+}
+
+macro_rules! to_string {
+ ($ARG:expr, $ARRAY:expr, $DELIMITER:expr, $NULL_STRING:expr, $WITH_NULL_STRING:expr, $ARRAY_TYPE:ident) => {{
+ let arr = downcast_arg!($ARRAY, $ARRAY_TYPE);
+ for x in arr {
+ match x {
+ Some(x) => {
+ $ARG.push_str(&x.to_string());
+ $ARG.push_str($DELIMITER);
+ }
+ None => {
+ if $WITH_NULL_STRING {
+ $ARG.push_str($NULL_STRING);
+ $ARG.push_str($DELIMITER);
+ }
+ }
+ }
+ }
+ Ok($ARG)
+ }};
+}
+
+macro_rules! call_array_function {
+ ($DATATYPE:expr, false) => {
+ match $DATATYPE {
+ DataType::Utf8 => array_function!(StringArray),
+ DataType::LargeUtf8 => array_function!(LargeStringArray),
+ DataType::Boolean => array_function!(BooleanArray),
+ DataType::Float32 => array_function!(Float32Array),
+ DataType::Float64 => array_function!(Float64Array),
+ DataType::Int8 => array_function!(Int8Array),
+ DataType::Int16 => array_function!(Int16Array),
+ DataType::Int32 => array_function!(Int32Array),
+ DataType::Int64 => array_function!(Int64Array),
+ DataType::UInt8 => array_function!(UInt8Array),
+ DataType::UInt16 => array_function!(UInt16Array),
+ DataType::UInt32 => array_function!(UInt32Array),
+ DataType::UInt64 => array_function!(UInt64Array),
+ _ => unreachable!(),
+ }
+ };
+ ($DATATYPE:expr, $INCLUDE_LIST:expr) => {{
+ match $DATATYPE {
+ DataType::List(_) => array_function!(ListArray),
+ DataType::Utf8 => array_function!(StringArray),
+ DataType::LargeUtf8 => array_function!(LargeStringArray),
+ DataType::Boolean => array_function!(BooleanArray),
+ DataType::Float32 => array_function!(Float32Array),
+ DataType::Float64 => array_function!(Float64Array),
+ DataType::Int8 => array_function!(Int8Array),
+ DataType::Int16 => array_function!(Int16Array),
+ DataType::Int32 => array_function!(Int32Array),
+ DataType::Int64 => array_function!(Int64Array),
+ DataType::UInt8 => array_function!(UInt8Array),
+ DataType::UInt16 => array_function!(UInt16Array),
+ DataType::UInt32 => array_function!(UInt32Array),
+ DataType::UInt64 => array_function!(UInt64Array),
+ _ => unreachable!(),
+ }
+ }};
+}
+
+/// Array_to_string SQL function
+pub(super) fn array_to_string(args: &[ArrayRef]) -> datafusion_common::Result<ArrayRef> {
+ if args.len() < 2 || args.len() > 3 {
+ return exec_err!("array_to_string expects two or three arguments");
+ }
+
+ let arr = &args[0];
+
+ let delimiters = as_string_array(&args[1])?;
+ let delimiters: Vec<Option<&str>> = delimiters.iter().collect();
+
+ let mut null_string = String::from("");
+ let mut with_null_string = false;
+ if args.len() == 3 {
+ null_string = as_string_array(&args[2])?.value(0).to_string();
+ with_null_string = true;
+ }
+
+ fn compute_array_to_string(
+ arg: &mut String,
+ arr: ArrayRef,
+ delimiter: String,
+ null_string: String,
+ with_null_string: bool,
+ ) -> datafusion_common::Result<&mut String> {
+ match arr.data_type() {
+ DataType::List(..) => {
+ let list_array = as_list_array(&arr)?;
+ for i in 0..list_array.len() {
+ compute_array_to_string(
+ arg,
+ list_array.value(i),
+ delimiter.clone(),
+ null_string.clone(),
+ with_null_string,
+ )?;
+ }
+
+ Ok(arg)
+ }
+ DataType::LargeList(..) => {
+ let list_array = as_large_list_array(&arr)?;
+ for i in 0..list_array.len() {
+ compute_array_to_string(
+ arg,
+ list_array.value(i),
+ delimiter.clone(),
+ null_string.clone(),
+ with_null_string,
+ )?;
+ }
+
+ Ok(arg)
+ }
+ DataType::Null => Ok(arg),
+ data_type => {
+ macro_rules! array_function {
+ ($ARRAY_TYPE:ident) => {
+ to_string!(
+ arg,
+ arr,
+ &delimiter,
+ &null_string,
+ with_null_string,
+ $ARRAY_TYPE
+ )
+ };
+ }
+ call_array_function!(data_type, false)
+ }
+ }
+ }
+
+ fn generate_string_array<O: OffsetSizeTrait>(
+ list_arr: &GenericListArray<O>,
+ delimiters: Vec<Option<&str>>,
+ null_string: String,
+ with_null_string: bool,
+ ) -> datafusion_common::Result<StringArray> {
+ let mut res: Vec<Option<String>> = Vec::new();
+ for (arr, &delimiter) in list_arr.iter().zip(delimiters.iter()) {
+ if let (Some(arr), Some(delimiter)) = (arr, delimiter) {
+ let mut arg = String::from("");
+ let s = compute_array_to_string(
+ &mut arg,
+ arr,
+ delimiter.to_string(),
+ null_string.clone(),
+ with_null_string,
+ )?
+ .clone();
+
+ if let Some(s) = s.strip_suffix(delimiter) {
+ res.push(Some(s.to_string()));
+ } else {
+ res.push(Some(s));
+ }
+ } else {
+ res.push(None);
+ }
+ }
+
+ Ok(StringArray::from(res))
+ }
+
+ let arr_type = arr.data_type();
+ let string_arr = match arr_type {
+ DataType::List(_) | DataType::FixedSizeList(_, _) => {
+ let list_array = as_list_array(&arr)?;
+ generate_string_array::<i32>(
+ list_array,
+ delimiters,
+ null_string,
+ with_null_string,
+ )?
+ }
+ DataType::LargeList(_) => {
+ let list_array = as_large_list_array(&arr)?;
+ generate_string_array::<i64>(
+ list_array,
+ delimiters,
+ null_string,
+ with_null_string,
+ )?
+ }
+ _ => {
+ let mut arg = String::from("");
+ let mut res: Vec<Option<String>> = Vec::new();
+ // delimiter length is 1
+ assert_eq!(delimiters.len(), 1);
+ let delimiter = delimiters[0].unwrap();
+ let s = compute_array_to_string(
+ &mut arg,
+ arr.clone(),
+ delimiter.to_string(),
+ null_string,
+ with_null_string,
+ )?
+ .clone();
+
+ if !s.is_empty() {
+ let s = s.strip_suffix(delimiter).unwrap().to_string();
+ res.push(Some(s));
+ } else {
+ res.push(Some(s));
+ }
+ StringArray::from(res)
+ }
+ };
+
+ Ok(Arc::new(string_arr))
+}
diff --git a/datafusion/functions-array/src/lib.rs b/datafusion/functions-array/src/lib.rs
new file mode 100644
index 0000000000..84997ed10e
--- /dev/null
+++ b/datafusion/functions-array/src/lib.rs
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Array Functions for [DataFusion].
+//!
+//! This crate contains a collection of array functions implemented using the
+//! extension API.
+//!
+//! [DataFusion]: https://crates.io/crates/datafusion
+//!
+//! You can register the functions in this crate using the [`register_all`] function.
+//!
+
+#[macro_use]
+pub mod macros;
+
+mod kernels;
+mod udf;
+
+use datafusion_common::Result;
+use datafusion_execution::FunctionRegistry;
+use datafusion_expr::ScalarUDF;
+use log::debug;
+use std::sync::Arc;
+
+/// Fluent-style API for creating `Expr`s
+pub mod expr_fn {
+ pub use super::udf::array_to_string;
+}
+
+/// Registers all enabled packages with a [`FunctionRegistry`]
+pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> {
+ let functions: Vec<Arc<ScalarUDF>> = vec![udf::array_to_string_udf()];
+ functions.into_iter().try_for_each(|udf| {
+ let existing_udf = registry.register_udf(udf)?;
+ if let Some(existing_udf) = existing_udf {
+ debug!("Overwrite existing UDF: {}", existing_udf.name());
+ }
+ Ok(()) as Result<()>
+ })?;
+ Ok(())
+}
diff --git a/datafusion/functions-array/src/macros.rs b/datafusion/functions-array/src/macros.rs
new file mode 100644
index 0000000000..ef25a8d5c6
--- /dev/null
+++ b/datafusion/functions-array/src/macros.rs
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Creates external API functions for an array UDF. Specifically, creates
+///
+/// 1. Single `ScalarUDF` instance
+///
+/// Creates a singleton `ScalarUDF` of the `$UDF` function named `$GNAME` and a
+/// function named `$NAME` which returns that function named $NAME.
+///
+/// This is used to ensure creating the list of `ScalarUDF` only happens once.
+///
+/// # 2. `expr_fn` style function
+///
+/// These are functions that create an `Expr` that invokes the UDF, used
+/// primarily to programmatically create expressions.
+///
+/// For example:
+/// ```text
+/// pub fn array_to_string(delimiter: Expr) -> Expr {
+/// ...
+/// }
+/// ```
+/// # Arguments
+/// * `UDF`: name of the [`ScalarUDFImpl`]
+/// * `EXPR_FN`: name of the expr_fn function to be created
+/// * `arg`: 0 or more named arguments for the function
+/// * `DOC`: documentation string for the function
+/// * `SCALAR_UDF_FUNC`: name of the function to create (just) the `ScalarUDF`
+/// * `GNAME`: name for the single static instance of the `ScalarUDF`
+macro_rules! make_udf_function {
+ ($UDF:ty, $EXPR_FN:ident, $($arg:ident)*, $DOC:expr , $SCALAR_UDF_FN:ident) => {
+ paste::paste! {
+ // "fluent expr_fn" style function
+ #[doc = $DOC]
+ pub fn $EXPR_FN($($arg: Expr),*) -> Expr {
+ Expr::ScalarFunction(ScalarFunction::new_udf(
+ $SCALAR_UDF_FN(),
+ vec![$($arg),*],
+ ))
+ }
+
+ /// Singleton instance of [`$UDF`], ensures the UDF is only created once
+ /// named STATIC_$(UDF). For example `STATIC_ArrayToString`
+ #[allow(non_upper_case_globals)]
+ static [< STATIC_ $UDF >]: std::sync::OnceLock<std::sync::Arc<datafusion_expr::ScalarUDF>> =
+ std::sync::OnceLock::new();
+
+ /// ScalarFunction that returns a [`ScalarUDF`] for [`$UDF`]
+ ///
+ /// [`ScalarUDF`]: datafusion_expr::ScalarUDF
+ pub fn $SCALAR_UDF_FN() -> std::sync::Arc<datafusion_expr::ScalarUDF> {
+ [< STATIC_ $UDF >]
+ .get_or_init(|| {
+ std::sync::Arc::new(datafusion_expr::ScalarUDF::new_from_impl(
+ <$UDF>::new(),
+ ))
+ })
+ .clone()
+ }
+ }
+ };
+}
diff --git a/datafusion/functions-array/src/udf.rs b/datafusion/functions-array/src/udf.rs
new file mode 100644
index 0000000000..b7f9d2497f
--- /dev/null
+++ b/datafusion/functions-array/src/udf.rs
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`ScalarUDFImpl`] definitions for array functions.
+
+use arrow::datatypes::DataType;
+use datafusion_common::{plan_err, DataFusionError};
+use datafusion_expr::expr::ScalarFunction;
+use datafusion_expr::Expr;
+use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};
+use std::any::Any;
+
+// Create static instances of ScalarUDFs for each function
+make_udf_function!(ArrayToString,
+ array_to_string,
+ array delimiter, // arg name
+ "converts each element to its text representation.", // doc
+ array_to_string_udf // internal function name
+);
+
+#[derive(Debug)]
+pub(super) struct ArrayToString {
+ signature: Signature,
+ aliases: Vec<String>,
+}
+
+impl ArrayToString {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ aliases: vec![
+ String::from("array_to_string"),
+ String::from("list_to_string"),
+ String::from("array_join"),
+ String::from("list_join"),
+ ],
+ }
+ }
+}
+
+impl ScalarUDFImpl for ArrayToString {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+ fn name(&self) -> &str {
+ "array_to_string"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result<DataType> {
+ use DataType::*;
+ Ok(match arg_types[0] {
+ List(_) | LargeList(_) | FixedSizeList(_, _) => Utf8,
+ _ => {
+ return plan_err!("The array_to_string function can only accept List/LargeList/FixedSizeList.");
+ }
+ })
+ }
+
+ fn invoke(&self, args: &[ColumnarValue]) -> datafusion_common::Result<ColumnarValue> {
+ let args = ColumnarValue::values_to_arrays(args)?;
+ crate::kernels::array_to_string(&args).map(ColumnarValue::Array)
+ }
+
+ fn aliases(&self) -> &[String] {
+ &self.aliases
+ }
+}
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 1c12289491..6a32e8299a 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -1376,7 +1376,6 @@ mod tests {
use datafusion_physical_expr::execution_props::ExecutionProps;
use chrono::{DateTime, TimeZone, Utc};
- use datafusion_physical_expr::functions::columnar_values_to_array;
// ------------------------------
// --- ExprSimplifier tests -----
@@ -1490,7 +1489,7 @@ mod tests {
let return_type = Arc::new(DataType::Int32);
let fun = Arc::new(|args: &[ColumnarValue]| {
- let args = columnar_values_to_array(args)?;
+ let args = ColumnarValue::values_to_arrays(args)?;
let arg0 = as_int32_array(&args[0])?;
let arg1 = as_int32_array(&args[1])?;
diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs
index 844dae0917..e17d2b210e 100644
--- a/datafusion/physical-expr/src/array_expressions.rs
+++ b/datafusion/physical-expr/src/array_expressions.rs
@@ -229,46 +229,6 @@ fn check_datatypes(name: &str, args: &[&ArrayRef]) -> Result<()> {
Ok(())
}
-macro_rules! call_array_function {
- ($DATATYPE:expr, false) => {
- match $DATATYPE {
- DataType::Utf8 => array_function!(StringArray),
- DataType::LargeUtf8 => array_function!(LargeStringArray),
- DataType::Boolean => array_function!(BooleanArray),
- DataType::Float32 => array_function!(Float32Array),
- DataType::Float64 => array_function!(Float64Array),
- DataType::Int8 => array_function!(Int8Array),
- DataType::Int16 => array_function!(Int16Array),
- DataType::Int32 => array_function!(Int32Array),
- DataType::Int64 => array_function!(Int64Array),
- DataType::UInt8 => array_function!(UInt8Array),
- DataType::UInt16 => array_function!(UInt16Array),
- DataType::UInt32 => array_function!(UInt32Array),
- DataType::UInt64 => array_function!(UInt64Array),
- _ => unreachable!(),
- }
- };
- ($DATATYPE:expr, $INCLUDE_LIST:expr) => {{
- match $DATATYPE {
- DataType::List(_) => array_function!(ListArray),
- DataType::Utf8 => array_function!(StringArray),
- DataType::LargeUtf8 => array_function!(LargeStringArray),
- DataType::Boolean => array_function!(BooleanArray),
- DataType::Float32 => array_function!(Float32Array),
- DataType::Float64 => array_function!(Float64Array),
- DataType::Int8 => array_function!(Int8Array),
- DataType::Int16 => array_function!(Int16Array),
- DataType::Int32 => array_function!(Int32Array),
- DataType::Int64 => array_function!(Int64Array),
- DataType::UInt8 => array_function!(UInt8Array),
- DataType::UInt16 => array_function!(UInt16Array),
- DataType::UInt32 => array_function!(UInt32Array),
- DataType::UInt64 => array_function!(UInt64Array),
- _ => unreachable!(),
- }
- }};
-}
-
/// Convert one or more [`ArrayRef`] of the same type into a
/// `ListArray` or 'LargeListArray' depending on the offset size.
///
@@ -1870,27 +1830,6 @@ pub fn array_replace_all(args: &[ArrayRef]) -> Result<ArrayRef> {
}
}
-macro_rules! to_string {
- ($ARG:expr, $ARRAY:expr, $DELIMITER:expr, $NULL_STRING:expr, $WITH_NULL_STRING:expr, $ARRAY_TYPE:ident) => {{
- let arr = downcast_arg!($ARRAY, $ARRAY_TYPE);
- for x in arr {
- match x {
- Some(x) => {
- $ARG.push_str(&x.to_string());
- $ARG.push_str($DELIMITER);
- }
- None => {
- if $WITH_NULL_STRING {
- $ARG.push_str($NULL_STRING);
- $ARG.push_str($DELIMITER);
- }
- }
- }
- }
- Ok($ARG)
- }};
-}
-
#[derive(Debug, PartialEq)]
enum SetOp {
Union,
@@ -2058,159 +1997,6 @@ pub fn array_intersect(args: &[ArrayRef]) -> Result<ArrayRef> {
general_set_op(array1, array2, SetOp::Intersect)
}
-/// Array_to_string SQL function
-pub fn array_to_string(args: &[ArrayRef]) -> Result<ArrayRef> {
- if args.len() < 2 || args.len() > 3 {
- return exec_err!("array_to_string expects two or three arguments");
- }
-
- let arr = &args[0];
-
- let delimiters = as_string_array(&args[1])?;
- let delimiters: Vec<Option<&str>> = delimiters.iter().collect();
-
- let mut null_string = String::from("");
- let mut with_null_string = false;
- if args.len() == 3 {
- null_string = as_string_array(&args[2])?.value(0).to_string();
- with_null_string = true;
- }
-
- fn compute_array_to_string(
- arg: &mut String,
- arr: ArrayRef,
- delimiter: String,
- null_string: String,
- with_null_string: bool,
- ) -> Result<&mut String> {
- match arr.data_type() {
- DataType::List(..) => {
- let list_array = as_list_array(&arr)?;
- for i in 0..list_array.len() {
- compute_array_to_string(
- arg,
- list_array.value(i),
- delimiter.clone(),
- null_string.clone(),
- with_null_string,
- )?;
- }
-
- Ok(arg)
- }
- DataType::LargeList(..) => {
- let list_array = as_large_list_array(&arr)?;
- for i in 0..list_array.len() {
- compute_array_to_string(
- arg,
- list_array.value(i),
- delimiter.clone(),
- null_string.clone(),
- with_null_string,
- )?;
- }
-
- Ok(arg)
- }
- DataType::Null => Ok(arg),
- data_type => {
- macro_rules! array_function {
- ($ARRAY_TYPE:ident) => {
- to_string!(
- arg,
- arr,
- &delimiter,
- &null_string,
- with_null_string,
- $ARRAY_TYPE
- )
- };
- }
- call_array_function!(data_type, false)
- }
- }
- }
-
- fn generate_string_array<O: OffsetSizeTrait>(
- list_arr: &GenericListArray<O>,
- delimiters: Vec<Option<&str>>,
- null_string: String,
- with_null_string: bool,
- ) -> Result<StringArray> {
- let mut res: Vec<Option<String>> = Vec::new();
- for (arr, &delimiter) in list_arr.iter().zip(delimiters.iter()) {
- if let (Some(arr), Some(delimiter)) = (arr, delimiter) {
- let mut arg = String::from("");
- let s = compute_array_to_string(
- &mut arg,
- arr,
- delimiter.to_string(),
- null_string.clone(),
- with_null_string,
- )?
- .clone();
-
- if let Some(s) = s.strip_suffix(delimiter) {
- res.push(Some(s.to_string()));
- } else {
- res.push(Some(s));
- }
- } else {
- res.push(None);
- }
- }
-
- Ok(StringArray::from(res))
- }
-
- let arr_type = arr.data_type();
- let string_arr = match arr_type {
- DataType::List(_) | DataType::FixedSizeList(_, _) => {
- let list_array = as_list_array(&arr)?;
- generate_string_array::<i32>(
- list_array,
- delimiters,
- null_string,
- with_null_string,
- )?
- }
- DataType::LargeList(_) => {
- let list_array = as_large_list_array(&arr)?;
- generate_string_array::<i64>(
- list_array,
- delimiters,
- null_string,
- with_null_string,
- )?
- }
- _ => {
- let mut arg = String::from("");
- let mut res: Vec<Option<String>> = Vec::new();
- // delimiter length is 1
- assert_eq!(delimiters.len(), 1);
- let delimiter = delimiters[0].unwrap();
- let s = compute_array_to_string(
- &mut arg,
- arr.clone(),
- delimiter.to_string(),
- null_string,
- with_null_string,
- )?
- .clone();
-
- if !s.is_empty() {
- let s = s.strip_suffix(delimiter).unwrap().to_string();
- res.push(Some(s));
- } else {
- res.push(Some(s));
- }
- StringArray::from(res)
- }
- };
-
- Ok(Arc::new(string_arr))
-}
-
/// Cardinality SQL function
pub fn cardinality(args: &[ArrayRef]) -> Result<ArrayRef> {
if args.len() != 1 {
diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs
index 21eaeab721..bd5c1677b0 100644
--- a/datafusion/physical-expr/src/functions.rs
+++ b/datafusion/physical-expr/src/functions.rs
@@ -173,49 +173,9 @@ pub(crate) enum Hint {
AcceptsSingular,
}
-/// A helper function used to infer the length of arguments of Scalar functions and convert
-/// [`ColumnarValue`]s to [`ArrayRef`]s with the inferred length. Note that this function
-/// only works for functions that accept either that all arguments are scalars or all arguments
-/// are arrays with same length. Otherwise, it will return an error.
+#[deprecated(since = "36.0.0", note = "Use ColumarValue::values_to_arrays instead")]
pub fn columnar_values_to_array(args: &[ColumnarValue]) -> Result<Vec<ArrayRef>> {
- if args.is_empty() {
- return Ok(vec![]);
- }
-
- let len = args
- .iter()
- .fold(Option::<usize>::None, |acc, arg| match arg {
- ColumnarValue::Scalar(_) if acc.is_none() => Some(1),
- ColumnarValue::Scalar(_) => {
- if let Some(1) = acc {
- acc
- } else {
- None
- }
- }
- ColumnarValue::Array(a) => {
- if let Some(l) = acc {
- if l == a.len() {
- acc
- } else {
- None
- }
- } else {
- Some(a.len())
- }
- }
- });
-
- let inferred_length = len.ok_or(DataFusionError::Internal(
- "Arguments has mixed length".to_string(),
- ))?;
-
- let args = args
- .iter()
- .map(|arg| arg.clone().into_array(inferred_length))
- .collect::<Result<Vec<_>>>()?;
-
- Ok(args)
+ ColumnarValue::values_to_arrays(args)
}
/// Decorates a function to handle [`ScalarValue`]s by converting them to arrays before calling the function
@@ -451,9 +411,6 @@ pub fn create_physical_fun(
BuiltinScalarFunction::ArraySlice => Arc::new(|args| {
make_scalar_function_inner(array_expressions::array_slice)(args)
}),
- BuiltinScalarFunction::ArrayToString => Arc::new(|args| {
- make_scalar_function_inner(array_expressions::array_to_string)(args)
- }),
BuiltinScalarFunction::ArrayIntersect => Arc::new(|args| {
make_scalar_function_inner(array_expressions::array_intersect)(args)
}),
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index f2b5c5dd42..61f8c7f5de 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -635,7 +635,7 @@ enum ScalarFunction {
ArrayPrepend = 94;
ArrayRemove = 95;
ArrayReplace = 96;
- ArrayToString = 97;
+ // 97 was ArrayToString
Cardinality = 98;
ArrayElement = 99;
ArraySlice = 100;
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index b9a8c5fc07..d9135c1dca 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -22387,7 +22387,6 @@ impl serde::Serialize for ScalarFunction {
Self::ArrayPrepend => "ArrayPrepend",
Self::ArrayRemove => "ArrayRemove",
Self::ArrayReplace => "ArrayReplace",
- Self::ArrayToString => "ArrayToString",
Self::Cardinality => "Cardinality",
Self::ArrayElement => "ArrayElement",
Self::ArraySlice => "ArraySlice",
@@ -22531,7 +22530,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction {
"ArrayPrepend",
"ArrayRemove",
"ArrayReplace",
- "ArrayToString",
"Cardinality",
"ArrayElement",
"ArraySlice",
@@ -22704,7 +22702,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction {
"ArrayPrepend" => Ok(ScalarFunction::ArrayPrepend),
"ArrayRemove" => Ok(ScalarFunction::ArrayRemove),
"ArrayReplace" => Ok(ScalarFunction::ArrayReplace),
- "ArrayToString" => Ok(ScalarFunction::ArrayToString),
"Cardinality" => Ok(ScalarFunction::Cardinality),
"ArrayElement" => Ok(ScalarFunction::ArrayElement),
"ArraySlice" => Ok(ScalarFunction::ArraySlice),
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 758ef2dcb5..342efa6f1d 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -2730,7 +2730,7 @@ pub enum ScalarFunction {
ArrayPrepend = 94,
ArrayRemove = 95,
ArrayReplace = 96,
- ArrayToString = 97,
+ /// reserved = 97; // was ArrayToString
Cardinality = 98,
ArrayElement = 99,
ArraySlice = 100,
@@ -2871,7 +2871,6 @@ impl ScalarFunction {
ScalarFunction::ArrayPrepend => "ArrayPrepend",
ScalarFunction::ArrayRemove => "ArrayRemove",
ScalarFunction::ArrayReplace => "ArrayReplace",
- ScalarFunction::ArrayToString => "ArrayToString",
ScalarFunction::Cardinality => "Cardinality",
ScalarFunction::ArrayElement => "ArrayElement",
ScalarFunction::ArraySlice => "ArraySlice",
@@ -3009,7 +3008,6 @@ impl ScalarFunction {
"ArrayPrepend" => Some(Self::ArrayPrepend),
"ArrayRemove" => Some(Self::ArrayRemove),
"ArrayReplace" => Some(Self::ArrayReplace),
- "ArrayToString" => Some(Self::ArrayToString),
"Cardinality" => Some(Self::Cardinality),
"ArrayElement" => Some(Self::ArrayElement),
"ArraySlice" => Some(Self::ArraySlice),
diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs
index decf3b1874..f7e509885b 100644
--- a/datafusion/proto/src/logical_plan/from_proto.rs
+++ b/datafusion/proto/src/logical_plan/from_proto.rs
@@ -51,11 +51,10 @@ use datafusion_expr::{
array_intersect, array_length, array_ndims, array_pop_back, array_pop_front,
array_position, array_positions, array_prepend, array_remove, array_remove_all,
array_remove_n, array_repeat, array_replace, array_replace_all, array_replace_n,
- array_resize, array_slice, array_sort, array_to_string, array_union, arrow_typeof,
- ascii, asin, asinh, atan, atan2, atanh, bit_length, btrim, cardinality, cbrt, ceil,
- character_length, chr, coalesce, concat_expr, concat_ws_expr, cos, cosh, cot,
- current_date, current_time, date_bin, date_part, date_trunc, degrees, digest,
- ends_with, exp,
+ array_resize, array_slice, array_sort, array_union, arrow_typeof, ascii, asin, asinh,
+ atan, atan2, atanh, bit_length, btrim, cardinality, cbrt, ceil, character_length,
+ chr, coalesce, concat_expr, concat_ws_expr, cos, cosh, cot, current_date,
+ current_time, date_bin, date_part, date_trunc, degrees, digest, ends_with, exp,
expr::{self, InList, Sort, WindowFunction},
factorial, find_in_set, flatten, floor, from_unixtime, gcd, gen_range, initcap,
instr, isnan, iszero, lcm, left, levenshtein, ln, log, log10, log2,
@@ -506,7 +505,6 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction {
ScalarFunction::ArrayReplaceAll => Self::ArrayReplaceAll,
ScalarFunction::ArrayReverse => Self::ArrayReverse,
ScalarFunction::ArraySlice => Self::ArraySlice,
- ScalarFunction::ArrayToString => Self::ArrayToString,
ScalarFunction::ArrayIntersect => Self::ArrayIntersect,
ScalarFunction::ArrayUnion => Self::ArrayUnion,
ScalarFunction::ArrayResize => Self::ArrayResize,
@@ -1471,10 +1469,6 @@ pub fn parse_expr(
parse_expr(&args[2], registry)?,
parse_expr(&args[3], registry)?,
)),
- ScalarFunction::ArrayToString => Ok(array_to_string(
- parse_expr(&args[0], registry)?,
- parse_expr(&args[1], registry)?,
- )),
ScalarFunction::Range => Ok(gen_range(
args.to_owned()
.iter()
diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs
index e094994840..d389568116 100644
--- a/datafusion/proto/src/logical_plan/to_proto.rs
+++ b/datafusion/proto/src/logical_plan/to_proto.rs
@@ -1502,7 +1502,6 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction {
BuiltinScalarFunction::ArrayReplaceAll => Self::ArrayReplaceAll,
BuiltinScalarFunction::ArrayReverse => Self::ArrayReverse,
BuiltinScalarFunction::ArraySlice => Self::ArraySlice,
- BuiltinScalarFunction::ArrayToString => Self::ArrayToString,
BuiltinScalarFunction::ArrayIntersect => Self::ArrayIntersect,
BuiltinScalarFunction::ArrayUnion => Self::ArrayUnion,
BuiltinScalarFunction::Range => Self::Range,