You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/05 00:00:44 UTC
(datafusion) branch gh-readonly-queue/main/pr-25544-c3ef346887412eeaeef644809c8fcfaf782f94e4 created (now 287b4da0e5)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25544-c3ef346887412eeaeef644809c8fcfaf782f94e4
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 287b4da0e5 fix: handle null lists with empty children in Spark's `shuffle` (#25544)
This branch includes the following new commits:
new 287b4da0e5 fix: handle null lists with empty children in Spark's `shuffle` (#25544)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: handle null lists with empty children in Spark's `shuffle` (#25544)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25544-c3ef346887412eeaeef644809c8fcfaf782f94e4
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 287b4da0e511473d267a5dc233b38b9836c13c49
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Mon Oct 5 00:00:19 2026 +0000
fix: handle null lists with empty children in Spark's `shuffle` (#25544)
## Which issue does this PR close?
- N/A
## Rationale for this change
The Spark-compatible `shuffle` implementation unnecessarily copied the
first child element as a placeholder for each null list. This is wrong
for typed-NULL lists with no children, and was redundant work anyway:
NULL lists can be adequately represented by a repeated offset entry,
along with setting the appropriate bit in the validity buffer.
## What changes are included in this PR?
* Fix bug
* Add unit test
## What is the testing strategy for this PR?
Existing tests pass, new test added.
## Are there any user-facing changes?
No.
---
datafusion/spark/src/function/array/shuffle.rs | 40 ++++++++++++++++++++--
.../test_files/spark/array/shuffle.slt | 6 ++++
2 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/datafusion/spark/src/function/array/shuffle.rs b/datafusion/spark/src/function/array/shuffle.rs
index c05fcda77b..cf1bfe4ed8 100644
--- a/datafusion/spark/src/function/array/shuffle.rs
+++ b/datafusion/spark/src/function/array/shuffle.rs
@@ -187,8 +187,7 @@ fn general_array_shuffle<O: OffsetSizeTrait>(
// skip the null value
if array.is_null(row_index) {
nulls.push(false);
- offsets.push(offsets[row_index] + O::one());
- mutable.try_extend(0, 0, 1)?;
+ offsets.push(offsets[row_index]);
continue;
}
nulls.push(true);
@@ -272,9 +271,46 @@ fn fixed_size_array_shuffle(
#[cfg(test)]
mod tests {
use super::*;
+ use arrow::array::{Int32Array, ListArray};
+ use arrow::buffer::NullBuffer;
use arrow::datatypes::Field;
use datafusion_expr::ReturnFieldArgs;
+ #[test]
+ fn test_shuffle_null_lists() -> Result<()> {
+ let field = Arc::new(Field::new_list_field(DataType::Int32, false));
+ let input = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::new(vec![0, 0, 0, 1, 1].into()),
+ Arc::new(Int32Array::from(vec![42])),
+ Some(NullBuffer::from(vec![false, true, true, false])),
+ );
+ let empty_child = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::new(vec![0, 0, 0].into()),
+ Arc::new(Int32Array::from(Vec::<i32>::new())),
+ Some(NullBuffer::from(vec![false, true])),
+ );
+
+ for (list, expected_values_len) in
+ [(empty_child, 0), (input.clone(), 1), (input.slice(3, 1), 0)]
+ {
+ for data_type in [list.data_type().clone(), LargeList(Arc::clone(&field))] {
+ let array = arrow::compute::cast(&list, &data_type)?;
+ let result = array_shuffle_with_seed(&[Arc::clone(&array)], Some(0))?;
+ assert_eq!(result.as_ref(), array.as_ref());
+ // Null rows need no placeholder child value.
+ let values_len = match data_type {
+ List(_) => as_list_array(&result)?.values().len(),
+ LargeList(_) => as_large_list_array(&result)?.values().len(),
+ _ => unreachable!(),
+ };
+ assert_eq!(values_len, expected_values_len);
+ }
+ }
+ Ok(())
+ }
+
#[test]
fn test_shuffle_nullability() {
let shuffle = SparkShuffle::new();
diff --git a/datafusion/sqllogictest/test_files/spark/array/shuffle.slt b/datafusion/sqllogictest/test_files/spark/array/shuffle.slt
index 061d89c4fa..b0acee18fd 100644
--- a/datafusion/sqllogictest/test_files/spark/array/shuffle.slt
+++ b/datafusion/sqllogictest/test_files/spark/array/shuffle.slt
@@ -45,6 +45,12 @@ SELECT shuffle(NULL);
----
NULL
+# Typed null lists have no child values to shuffle.
+query ??
+SELECT shuffle(arrow_cast(NULL, 'List(Int32)')), shuffle(arrow_cast(NULL, 'LargeList(Int32)'));
+----
+NULL NULL
+
# Test shuffle function with fixed size list arrays
query ?
SELECT shuffle(arrow_cast([1, 2, NULL, 3, 4, 5], 'FixedSizeList(6, Int64)'), 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org