You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2023/06/14 20:28:03 UTC
[arrow-datafusion] branch main updated: Support internal cast for BuiltinScalarFunction::MakeArray (#6607)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new e9fae9889d Support internal cast for BuiltinScalarFunction::MakeArray (#6607)
e9fae9889d is described below
commit e9fae9889d2a2e12d36ac3ca6a57e483e568d3cb
Author: Jay Zhan <ja...@gmail.com>
AuthorDate: Thu Jun 15 04:27:57 2023 +0800
Support internal cast for BuiltinScalarFunction::MakeArray (#6607)
* Internal cast for array()
Signed-off-by: jayzhan211 <ja...@gmail.com>
* Add sqllogictest
Signed-off-by: jayzhan211 <ja...@gmail.com>
* address CI fail
Signed-off-by: jayzhan211 <ja...@gmail.com>
* address comments
Signed-off-by: jayzhan211 <ja...@gmail.com>
* refactor
Signed-off-by: jayzhan211 <ja...@gmail.com>
* add schema for coerce_args_for_fun
Signed-off-by: jayzhan211 <ja...@gmail.com>
* add more tests
Signed-off-by: jayzhan211 <ja...@gmail.com>
---------
Signed-off-by: jayzhan211 <ja...@gmail.com>
---
datafusion-cli/Cargo.lock | 89 ++++++++--------------
.../core/tests/sqllogictests/test_files/array.slt | 36 +++++++++
datafusion/optimizer/src/analyzer/type_coercion.rs | 44 +++++++++--
3 files changed, 108 insertions(+), 61 deletions(-)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 71b18f71a5..9f6296b5b0 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -657,9 +657,9 @@ dependencies = [
[[package]]
name = "blake3"
-version = "1.3.3"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef"
+checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888"
dependencies = [
"arrayref",
"arrayvec",
@@ -883,9 +883,9 @@ dependencies = [
[[package]]
name = "constant_time_eq"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b"
+checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6"
[[package]]
name = "core-foundation"
@@ -1434,9 +1434,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.2.9"
+version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
@@ -1634,14 +1634,14 @@ dependencies = [
"hyper",
"rustls 0.21.1",
"tokio",
- "tokio-rustls 0.24.0",
+ "tokio-rustls 0.24.1",
]
[[package]]
name = "iana-time-zone"
-version = "0.1.56"
+version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c"
+checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -1855,9 +1855,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.18"
+version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
[[package]]
name = "lz4"
@@ -2101,9 +2101,9 @@ dependencies = [
[[package]]
name = "os_str_bytes"
-version = "6.5.0"
+version = "6.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267"
+checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac"
[[package]]
name = "outref"
@@ -2131,7 +2131,7 @@ dependencies = [
"libc",
"redox_syscall 0.3.5",
"smallvec",
- "windows-targets 0.48.0",
+ "windows-targets",
]
[[package]]
@@ -2313,9 +2313,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.59"
+version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b"
+checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406"
dependencies = [
"unicode-ident",
]
@@ -2454,7 +2454,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"tokio",
- "tokio-rustls 0.24.0",
+ "tokio-rustls 0.24.1",
"tokio-util",
"tower-service",
"url",
@@ -2492,9 +2492,9 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.37.19"
+version = "0.37.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
+checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0"
dependencies = [
"bitflags",
"errno",
@@ -2665,18 +2665,18 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc"
[[package]]
name = "serde"
-version = "1.0.163"
+version = "1.0.164"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
+checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.163"
+version = "1.0.164"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
+checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
dependencies = [
"proc-macro2",
"quote",
@@ -2873,15 +2873,16 @@ dependencies = [
[[package]]
name = "tempfile"
-version = "3.5.0"
+version = "3.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
+checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
dependencies = [
+ "autocfg",
"cfg-if",
"fastrand",
"redox_syscall 0.3.5",
"rustix",
- "windows-sys 0.45.0",
+ "windows-sys 0.48.0",
]
[[package]]
@@ -2932,9 +2933,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.21"
+version = "0.3.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc"
+checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd"
dependencies = [
"serde",
"time-core",
@@ -3022,9 +3023,9 @@ dependencies = [
[[package]]
name = "tokio-rustls"
-version = "0.24.0"
+version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5"
+checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls 0.21.1",
"tokio",
@@ -3392,7 +3393,7 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
- "windows-targets 0.48.0",
+ "windows-targets",
]
[[package]]
@@ -3410,37 +3411,13 @@ dependencies = [
"windows_x86_64_msvc 0.42.2",
]
-[[package]]
-name = "windows-sys"
-version = "0.45.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
- "windows-targets 0.48.0",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
-dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
+ "windows-targets",
]
[[package]]
diff --git a/datafusion/core/tests/sqllogictests/test_files/array.slt b/datafusion/core/tests/sqllogictests/test_files/array.slt
index 1835221380..459046136b 100644
--- a/datafusion/core/tests/sqllogictests/test_files/array.slt
+++ b/datafusion/core/tests/sqllogictests/test_files/array.slt
@@ -300,3 +300,39 @@ query II rowsort
select array_ndims(make_array()), array_ndims(make_array(make_array()))
----
1 2
+
+query ?
+select make_array(1, 2.0)
+----
+[1.0, 2.0]
+
+query ?
+select make_array(null, 1.0)
+----
+[, 1.0]
+
+query ?
+select make_array(1, 2.0, null, 3)
+----
+[1.0, 2.0, , 3.0]
+
+query ?
+select make_array(1.0, '2', null)
+----
+[1.0, 2, ]
+
+statement ok
+create table foo1 (x int, y double) as values (1, 2.0);
+
+query ?
+select make_array(x, y) from foo1;
+----
+[1.0, 2.0]
+
+statement ok
+create table foo2 (x float, y varchar) as values (1.0, '1');
+
+query ?
+select make_array(x, y) from foo2;
+----
+[1.0, 1]
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index e6023c4698..312136f4a7 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -42,8 +42,8 @@ use datafusion_expr::type_coercion::{is_datetime, is_numeric, is_utf8_or_large_u
use datafusion_expr::utils::from_plan;
use datafusion_expr::{
aggregate_function, is_false, is_not_false, is_not_true, is_not_unknown, is_true,
- is_unknown, type_coercion, AggregateFunction, Expr, LogicalPlan, Operator,
- Projection, WindowFrame, WindowFrameBound, WindowFrameUnits,
+ is_unknown, type_coercion, AggregateFunction, BuiltinScalarFunction, Expr,
+ LogicalPlan, Operator, Projection, WindowFrame, WindowFrameBound, WindowFrameUnits,
};
use datafusion_expr::{ExprSchemable, Signature};
@@ -388,13 +388,14 @@ impl TreeNodeRewriter for TypeCoercionRewriter {
Ok(expr)
}
Expr::ScalarFunction(ScalarFunction { fun, args }) => {
- let nex_expr = coerce_arguments_for_signature(
+ let new_args = coerce_arguments_for_signature(
args.as_slice(),
&self.schema,
&fun.signature(),
)?;
- let expr = Expr::ScalarFunction(ScalarFunction::new(fun, nex_expr));
- Ok(expr)
+ let new_args =
+ coerce_arguments_for_fun(new_args.as_slice(), &self.schema, &fun)?;
+ Ok(Expr::ScalarFunction(ScalarFunction::new(fun, new_args)))
}
Expr::AggregateFunction(expr::AggregateFunction {
fun,
@@ -603,6 +604,39 @@ fn coerce_arguments_for_signature(
.collect::<Result<Vec<_>>>()
}
+fn coerce_arguments_for_fun(
+ expressions: &[Expr],
+ schema: &DFSchema,
+ fun: &BuiltinScalarFunction,
+) -> Result<Vec<Expr>> {
+ if expressions.is_empty() {
+ return Ok(vec![]);
+ }
+
+ if *fun == BuiltinScalarFunction::MakeArray {
+ // Find the final data type for the function arguments
+ let current_types = expressions
+ .iter()
+ .map(|e| e.get_type(schema))
+ .collect::<Result<Vec<_>>>()?;
+
+ let new_type = current_types
+ .iter()
+ .skip(1)
+ .fold(current_types.first().unwrap().clone(), |acc, x| {
+ comparison_coercion(&acc, x).unwrap_or(acc)
+ });
+
+ return expressions
+ .iter()
+ .enumerate()
+ .map(|(_, expr)| cast_expr(expr, &new_type, schema))
+ .collect();
+ }
+
+ Ok(expressions.to_vec())
+}
+
/// Cast `expr` to the specified type, if possible
fn cast_expr(expr: &Expr, to_type: &DataType, schema: &DFSchema) -> Result<Expr> {
expr.clone().cast_to(to_type, schema)