You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/13 02:22:30 UTC
(datafusion-comet) 01/01: test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5839-7319eea04f1d08bf0c4ff74be5f522ffdec5b1be
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit fdf17164dea24144895080253e1edb5c17f337ae
Author: sam-1112 <75...@users.noreply.github.com>
AuthorDate: Sat Sep 12 21:17:24 2026 +0000
test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
Co-authored-by: Hung <hu...@Hungs-Mac-mini.local>
Co-authored-by: Andy Grove <ag...@apache.org>
---
native/spark-expr/src/array_funcs/array_slice.rs | 21 ++++++++++++++++++++-
.../apache/comet/CometArrayExpressionSuite.scala | 17 +++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/native/spark-expr/src/array_funcs/array_slice.rs b/native/spark-expr/src/array_funcs/array_slice.rs
index fcd3b467a6..28e1fc5185 100644
--- a/native/spark-expr/src/array_funcs/array_slice.rs
+++ b/native/spark-expr/src/array_funcs/array_slice.rs
@@ -188,6 +188,14 @@ mod tests {
use arrow::datatypes::{Field, Int32Type};
fn build_list(rows: Vec<Option<Vec<Option<i32>>>>) -> Arc<ListArray> {
+ let field = Arc::new(Field::new("item", DataType::Int32, true));
+ build_list_with_field(field, rows)
+ }
+
+ fn build_list_with_field(
+ field: FieldRef,
+ rows: Vec<Option<Vec<Option<i32>>>>,
+ ) -> Arc<ListArray> {
let mut offsets = vec![0i32];
let mut values: Vec<Option<i32>> = Vec::new();
let mut nulls = NullBufferBuilder::new(rows.len());
@@ -202,7 +210,6 @@ mod tests {
offsets.push(values.len() as i32);
}
let values = Arc::new(Int32Array::from(values)) as ArrayRef;
- let field = Arc::new(Field::new("item", DataType::Int32, true));
Arc::new(ListArray::new(
field,
OffsetBuffer::new(offsets.into()),
@@ -371,4 +378,16 @@ mod tests {
let length = Int64Array::from(vec![Some(-1)]);
assert!(slice_list::<i32>(list.as_ref(), &start, &length).is_err());
}
+
+ #[test]
+ fn preserves_non_nullable_element_field() {
+ let mut metadata = std::collections::HashMap::new();
+ metadata.insert("spark.element".to_string(), "non-null".to_string());
+ let field = Arc::new(Field::new("element", DataType::Int32, false).with_metadata(metadata));
+ let list = build_list_with_field(field, vec![Some(vec![Some(1), Some(2), Some(3)])]);
+ let start = Int64Array::from(vec![Some(1)]);
+ let length = Int64Array::from(vec![Some(2)]);
+ let result = slice_list::<i32>(list.as_ref(), &start, &length).unwrap();
+ assert_eq!(result.data_type(), list.data_type());
+ }
}
diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
index ad86dd15bc..8d0a04d7d6 100644
--- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
@@ -1489,6 +1489,23 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
}
}
+ // Spark declares split and sequence as ArrayType(..., containsNull=false). Native
+ // Parquet already normalizes stored children to nullable, so the non-null element
+ // field is produced after the scan. CometSlice must infer its return type from that
+ // input field; a planned nullable element disagrees with the kernel and crashes.
+ // https://github.com/apache/datafusion-comet/issues/5743
+ test("slice over expression-produced non-null element arrays (#5743)") {
+ val input = Seq((1, "axb", 2), (2, "", 3), (3, "cxd", 2))
+ withParquetDataFrame(input) { parquet =>
+ withParquetTable(parquet.toDF("id", "s", "n"), "t") {
+ checkSparkAnswerAndOperator(sql("SELECT id, slice(split(s, 'x'), 1, n) AS a FROM t"))
+ checkSparkAnswerAndOperator(sql("SELECT id, slice(sequence(1, n), 1, 2) AS a FROM t"))
+ checkSparkAnswerAndOperator(
+ sql("SELECT id, slice(concat(split(s, 'x'), array('z')), 1, n) AS a FROM t"))
+ }
+ }
+ }
+
// https://issues.apache.org/jira/browse/SPARK-55747
test("(ansi) GetArrayItem on null array from split()") {
withSQLConf(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org