You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/12 21:21:09 UTC
(datafusion) branch gh-readonly-queue/main/pr-25230-00c58dbf06cc7da750d7c14c5d6d12943fd0ae58 created (now 35bfe1b0a5)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25230-00c58dbf06cc7da750d7c14c5d6d12943fd0ae58
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 35bfe1b0a5 fix: Correct `array_repeat` NULL handling (#25230)
This branch includes the following new commits:
new 35bfe1b0a5 fix: Correct `array_repeat` NULL handling (#25230)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: Correct `array_repeat` NULL handling (#25230)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25230-00c58dbf06cc7da750d7c14c5d6d12943fd0ae58
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 35bfe1b0a5d675d6a2643e21d49fe8fa64c08fac
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Sat Sep 12 21:20:52 2026 +0000
fix: Correct `array_repeat` NULL handling (#25230)
## Which issue does this PR close?
- Closes #25223.
## Rationale for this change
Arrow allows elements of a `ListArray` to be logically NULL (according
to the validity buffer), but to have non-empty offset ranges. Such
elements should be considered NULL.
`array_repeat` only considered the offsets and ignored the validity
buffer, which resulted in mishandling such NULL values.
## What changes are included in this PR?
* Fix `array_repeat` bug
* Add unit test
* Add SLT
## What is the testing strategy for this PR?
Existing tests pass; new tests added. We add both a unit test and an SLT
because the SLT depends on `nullif` creating such a NULL value, which
might change in the future.
## Are there any user-facing changes?
Yes, bug fixed.
---
datafusion/functions-nested/src/repeat.rs | 53 +++++++++++++++++++---
.../sqllogictest/test_files/array/array_repeat.slt | 32 +++++++++++++
2 files changed, 79 insertions(+), 6 deletions(-)
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs
index d7dff21141..b275ae661e 100644
--- a/datafusion/functions-nested/src/repeat.rs
+++ b/datafusion/functions-nested/src/repeat.rs
@@ -262,8 +262,16 @@ fn general_list_repeat<O: OffsetSizeTrait>(
continue;
};
let list_is_valid = list_array.is_valid(row_idx);
- let start = list_offsets[row_idx].to_usize().unwrap();
- let end = list_offsets[row_idx + 1].to_usize().unwrap();
+ // A NULL list slot may still have a non-empty offset range; treat the
+ // range as empty to keep the inner offsets and taken values in step.
+ let (start, end) = if list_is_valid {
+ (
+ list_offsets[row_idx].to_usize().unwrap(),
+ list_offsets[row_idx + 1].to_usize().unwrap(),
+ )
+ } else {
+ (0, 0)
+ };
let row_len = end - start;
for _ in 0..count {
@@ -272,9 +280,7 @@ fn general_list_repeat<O: OffsetSizeTrait>(
let offset = checked_repeat_offset::<O>(inner_running)?;
inner_offsets.push(offset);
inner_nulls.append(list_is_valid);
- if list_is_valid {
- take_indices.extend(start as u64..end as u64);
- }
+ take_indices.extend(start as u64..end as u64);
}
}
@@ -388,7 +394,7 @@ mod tests {
use super::{array_repeat_inner, general_list_repeat, general_repeat};
use arrow::array::{Array, ArrayRef, AsArray, Int32Array, Int64Array, ListArray};
use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
- use arrow::datatypes::{Field, Int32Type};
+ use arrow::datatypes::{DataType, Field, Int32Type};
use datafusion_common::Result;
use std::sync::Arc;
@@ -445,6 +451,41 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_array_repeat_nested_null_list_with_nonempty_range() -> Result<()> {
+ // A NULL list slot might still have a non-empty offset range
+ // https://github.com/apache/datafusion/issues/25223
+ let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
+ let list_array = ListArray::new(
+ Arc::new(Field::new_list_field(DataType::Int32, true)),
+ OffsetBuffer::new(ScalarBuffer::from(vec![0, 3, 6])),
+ Arc::new(values),
+ Some(NullBuffer::from(vec![false, true])),
+ );
+ let counts = Int64Array::from(vec![2, 2]);
+
+ let result = general_list_repeat::<i32>(&list_array, &counts)?;
+ let repeated_values = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ None,
+ None,
+ Some(vec![Some(4), Some(5), Some(6)]),
+ Some(vec![Some(4), Some(5), Some(6)]),
+ ]);
+ let expected = ListArray::new(
+ Arc::new(Field::new_list_field(
+ repeated_values.data_type().clone(),
+ true,
+ )),
+ OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 4])),
+ Arc::new(repeated_values),
+ None,
+ );
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+
#[test]
fn scalar_count_exceeding_max_array_size_returns_error() {
let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));
diff --git a/datafusion/sqllogictest/test_files/array/array_repeat.slt b/datafusion/sqllogictest/test_files/array/array_repeat.slt
index 5073c7d4c5..97e55e7718 100644
--- a/datafusion/sqllogictest/test_files/array/array_repeat.slt
+++ b/datafusion/sqllogictest/test_files/array/array_repeat.slt
@@ -238,5 +238,37 @@ NULL [NULL, NULL, NULL]
statement ok
drop table array_repeat_large_nested_null_table
+# https://github.com/apache/datafusion/issues/25223
+statement ok
+create table array_repeat_null_slot_table
+as values
+([1, 2, 3], [1, 2, 3], 1),
+([4, 5, 6], [7], 2);
+
+query I?
+select column3, array_repeat(nullif(column1, column2), 2)
+from array_repeat_null_slot_table order by column3;
+----
+1 [NULL, NULL]
+2 [[4, 5, 6], [4, 5, 6]]
+
+query I?
+select
+ column3,
+ array_repeat(
+ nullif(
+ arrow_cast(column1, 'LargeList(Int64)'),
+ arrow_cast(column2, 'LargeList(Int64)')
+ ),
+ 2
+ )
+from array_repeat_null_slot_table order by column3;
+----
+1 [NULL, NULL]
+2 [[4, 5, 6], [4, 5, 6]]
+
+statement ok
+drop table array_repeat_null_slot_table
+
include ./cleanup.slt.part
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org