You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/16 07:21:36 UTC
(datafusion) branch main updated: fix: Fix Spark `slice` function `Null` type to `GenericListArray` casting issue (#20469)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new a0dbbab584 fix: Fix Spark `slice` function `Null` type to `GenericListArray` casting issue (#20469)
a0dbbab584 is described below
commit a0dbbab5849596ecb3db48d9e168f247155209e1
Author: Eren Avsarogullari <er...@apache.org>
AuthorDate: Thu Apr 16 00:19:33 2026 -0700
fix: Fix Spark `slice` function `Null` type to `GenericListArray` casting issue (#20469)
## Which issue does this PR close?
- Closes #20466.
## Rationale for this change
Currently, Spark `slice` function accepts Null Arrays and return `Null`
for this particular queries. DataFusion-Spark `slice` function also
needs to return `NULL` when Null Array is set.
**Spark Behavior** (tested with latest Spark master):
```
> SELECT slice(NULL, 1, 2);
+-----------------+
|slice(NULL, 1, 2)|
+-----------------+
| null|
+-----------------+
```
**DF Behaviour:**
Current:
```
query error
SELECT slice(NULL, 1, 2);
----
DataFusion error: Internal error: could not cast array of type Null to arrow_array::array::list_array::GenericListArray<i32>.
This issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues
```
New:
```
query ?
SELECT slice(NULL, 1, 2);
----
NULL
```
## What changes are included in this PR?
Explained under first section.
## Are these changes tested?
Added new UT cases for both `slice.rs` and `slice.slt`.
## Are there any user-facing changes?
Yes, currently, `slice` function returns error message for `Null` Array
inputs, however, expected behavior is to be returned `NULL` so end-user
will get expected result instead of error message.
---------
Co-authored-by: Martin Grigorov <ma...@users.noreply.github.com>
---
datafusion/spark/src/function/array/slice.rs | 85 ++++++++++++++++++++--
.../sqllogictest/test_files/spark/array/slice.slt | 23 ++++++
2 files changed, 102 insertions(+), 6 deletions(-)
diff --git a/datafusion/spark/src/function/array/slice.rs b/datafusion/spark/src/function/array/slice.rs
index d3884d8f3b..bcd10a1bf7 100644
--- a/datafusion/spark/src/function/array/slice.rs
+++ b/datafusion/spark/src/function/array/slice.rs
@@ -19,7 +19,9 @@ use arrow::array::{Array, ArrayRef, Int64Builder};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::cast::{as_int64_array, as_list_array};
use datafusion_common::utils::ListCoercion;
-use datafusion_common::{Result, exec_err, internal_err, utils::take_function_args};
+use datafusion_common::{
+ Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
+};
use datafusion_expr::{
ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs,
ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
@@ -78,17 +80,28 @@ impl ScalarUDFImpl for SparkSlice {
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
- Ok(Arc::new(Field::new(
- "slice",
- args.arg_fields[0].data_type().clone(),
- nullable,
- )))
+ let data_type = match args.arg_fields[0].data_type() {
+ DataType::Null => {
+ DataType::List(Arc::new(Field::new_list_field(DataType::Null, true)))
+ }
+ dt => dt.clone(),
+ };
+
+ Ok(Arc::new(Field::new("slice", data_type, nullable)))
}
fn invoke_with_args(
&self,
mut func_args: ScalarFunctionArgs,
) -> Result<ColumnarValue> {
+ if func_args.args[0].data_type() == DataType::Null {
+ return Ok(ColumnarValue::Scalar(ScalarValue::new_null_list(
+ DataType::Null,
+ true,
+ 1,
+ )));
+ }
+
let array_len = func_args
.args
.iter()
@@ -165,3 +178,63 @@ fn calculate_start_end(args: &[ArrayRef]) -> Result<(ArrayRef, ArrayRef)> {
Ok((Arc::new(adjusted_start.finish()), Arc::new(end.finish())))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::NullArray;
+ use arrow::datatypes::Field;
+ use datafusion_common::ScalarValue;
+ use datafusion_common::cast::as_list_array;
+ use datafusion_expr::ReturnFieldArgs;
+
+ #[test]
+ fn test_spark_slice_function_when_input_is_null() {
+ let slice = SparkSlice::new();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Arc::new(Field::new("a", DataType::Null, true)),
+ Arc::new(Field::new("s", DataType::Int64, true)),
+ Arc::new(Field::new("l", DataType::Int64, true)),
+ ];
+ let out = slice
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &arg_fields,
+ scalar_arguments: &[],
+ })
+ .unwrap();
+ assert_eq!(
+ out.data_type(),
+ &DataType::List(Arc::new(Field::new_list_field(DataType::Null, true)))
+ );
+ }
+
+ #[test]
+ fn test_spark_slice_function_when_input_array_is_null() {
+ let input_args = vec![
+ ColumnarValue::Array(Arc::new(NullArray::new(1))),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(3))),
+ ];
+
+ let args = ScalarFunctionArgs {
+ args: input_args,
+ arg_fields: vec![Arc::new(Field::new("item", DataType::Null, true))],
+ number_rows: 1,
+ return_field: Arc::new(Field::new(
+ "slice",
+ DataType::List(Arc::new(Field::new_list_field(DataType::Null, true))),
+ true,
+ )),
+ config_options: Arc::new(Default::default()),
+ };
+ let slice = SparkSlice::new();
+ let result = slice.invoke_with_args(args).unwrap();
+ let arr = result.to_array(1).unwrap();
+ let list = as_list_array(&arr).unwrap();
+ assert_eq!(
+ arr.data_type(),
+ &DataType::List(Arc::new(Field::new_list_field(DataType::Null, true)))
+ );
+ assert!(list.is_null(0));
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/array/slice.slt b/datafusion/sqllogictest/test_files/spark/array/slice.slt
index 4aba076aba..6dfc1c0c6d 100644
--- a/datafusion/sqllogictest/test_files/spark/array/slice.slt
+++ b/datafusion/sqllogictest/test_files/spark/array/slice.slt
@@ -114,3 +114,26 @@ query ?
SELECT slice([1, 2, 3, 4], CAST('2' AS INT), 4);
----
[2, 3, 4]
+
+query ?
+SELECT slice(column1, column2, column3)
+FROM VALUES
+(NULL, 1, 2),
+(NULL, 1, -2),
+(NULL, -1, 2),
+(NULL, 0, 2);
+----
+NULL
+NULL
+NULL
+NULL
+
+query ?
+SELECT slice(slice(NULL, 1, 2), 1, 2)
+----
+NULL
+
+query ?
+SELECT slice(slice(make_array(NULL), 1, 2), 1, 2)
+----
+[NULL]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org