You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/13 02:22:29 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-5839-7319eea04f1d08bf0c4ff74be5f522ffdec5b1be created (now fdf17164de)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-5839-7319eea04f1d08bf0c4ff74be5f522ffdec5b1be
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at fdf17164de test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
This branch includes the following new commits:
new fdf17164de test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5839-7319eea04f1d08bf0c4ff74be5f522ffdec5b1be
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit fdf17164dea24144895080253e1edb5c17f337ae
Author: sam-1112 <75...@users.noreply.github.com>
AuthorDate: Sat Sep 12 21:17:24 2026 +0000
test: cover slice over expression-produced non-null element arrays (#5743) (#5839)
Co-authored-by: Hung <hu...@Hungs-Mac-mini.local>
Co-authored-by: Andy Grove <ag...@apache.org>
---
native/spark-expr/src/array_funcs/array_slice.rs | 21 ++++++++++++++++++++-
.../apache/comet/CometArrayExpressionSuite.scala | 17 +++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/native/spark-expr/src/array_funcs/array_slice.rs b/native/spark-expr/src/array_funcs/array_slice.rs
index fcd3b467a6..28e1fc5185 100644
--- a/native/spark-expr/src/array_funcs/array_slice.rs
+++ b/native/spark-expr/src/array_funcs/array_slice.rs
@@ -188,6 +188,14 @@ mod tests {
use arrow::datatypes::{Field, Int32Type};
fn build_list(rows: Vec<Option<Vec<Option<i32>>>>) -> Arc<ListArray> {
+ let field = Arc::new(Field::new("item", DataType::Int32, true));
+ build_list_with_field(field, rows)
+ }
+
+ fn build_list_with_field(
+ field: FieldRef,
+ rows: Vec<Option<Vec<Option<i32>>>>,
+ ) -> Arc<ListArray> {
let mut offsets = vec![0i32];
let mut values: Vec<Option<i32>> = Vec::new();
let mut nulls = NullBufferBuilder::new(rows.len());
@@ -202,7 +210,6 @@ mod tests {
offsets.push(values.len() as i32);
}
let values = Arc::new(Int32Array::from(values)) as ArrayRef;
- let field = Arc::new(Field::new("item", DataType::Int32, true));
Arc::new(ListArray::new(
field,
OffsetBuffer::new(offsets.into()),
@@ -371,4 +378,16 @@ mod tests {
let length = Int64Array::from(vec![Some(-1)]);
assert!(slice_list::<i32>(list.as_ref(), &start, &length).is_err());
}
+
+ #[test]
+ fn preserves_non_nullable_element_field() {
+ let mut metadata = std::collections::HashMap::new();
+ metadata.insert("spark.element".to_string(), "non-null".to_string());
+ let field = Arc::new(Field::new("element", DataType::Int32, false).with_metadata(metadata));
+ let list = build_list_with_field(field, vec![Some(vec![Some(1), Some(2), Some(3)])]);
+ let start = Int64Array::from(vec![Some(1)]);
+ let length = Int64Array::from(vec![Some(2)]);
+ let result = slice_list::<i32>(list.as_ref(), &start, &length).unwrap();
+ assert_eq!(result.data_type(), list.data_type());
+ }
}
diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
index ad86dd15bc..8d0a04d7d6 100644
--- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
@@ -1489,6 +1489,23 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
}
}
+ // Spark declares split and sequence as ArrayType(..., containsNull=false). Native
+ // Parquet already normalizes stored children to nullable, so the non-null element
+ // field is produced after the scan. CometSlice must infer its return type from that
+ // input field; a planned nullable element disagrees with the kernel and crashes.
+ // https://github.com/apache/datafusion-comet/issues/5743
+ test("slice over expression-produced non-null element arrays (#5743)") {
+ val input = Seq((1, "axb", 2), (2, "", 3), (3, "cxd", 2))
+ withParquetDataFrame(input) { parquet =>
+ withParquetTable(parquet.toDF("id", "s", "n"), "t") {
+ checkSparkAnswerAndOperator(sql("SELECT id, slice(split(s, 'x'), 1, n) AS a FROM t"))
+ checkSparkAnswerAndOperator(sql("SELECT id, slice(sequence(1, n), 1, 2) AS a FROM t"))
+ checkSparkAnswerAndOperator(
+ sql("SELECT id, slice(concat(split(s, 'x'), array('z')), 1, n) AS a FROM t"))
+ }
+ }
+ }
+
// https://issues.apache.org/jira/browse/SPARK-55747
test("(ansi) GetArrayItem on null array from split()") {
withSQLConf(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org