You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/25 21:20:48 UTC
(datafusion) branch gh-readonly-queue/main/pr-23192-a0e9887550065324320c6fd52001aa23bae67485 created (now 476a76daeb)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23192-a0e9887550065324320c6fd52001aa23bae67485
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 476a76daeb fix: `array_compact` handle edge case with NULLs (#23192)
This branch includes the following new commits:
new 476a76daeb fix: `array_compact` handle edge case with NULLs (#23192)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: `array_compact` handle edge case with NULLs (#23192)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23192-a0e9887550065324320c6fd52001aa23bae67485
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 476a76daeb0c14538cbf2a04bbc3c99913f1d180
Author: Oleks V <co...@users.noreply.github.com>
AuthorDate: Thu Jun 25 14:20:27 2026 -0700
fix: `array_compact` handle edge case with NULLs (#23192)
## Rationale for this change
`array_compact(make_array(NULL, NULL, NULL))` returned `[NULL, NULL,
NULL]` instead of an empty array.
Root cause: `make_array(NULL, NULL, NULL)` has type `List(Null)`, whose
inner values are an Arrow `NullArray`. `NullArray::nulls()` returns
`None` (it has no validity buffer), so the default
`Array::is_null()` returns `false` for every index — even though every
element is logically null. The compaction loop saw "no nulls" and copied
all elements through unchanged.
## What changes are included in this PR?
- In `compact_list`, resolve the values' null mask once via
`values.logical_nulls()` and use that buffer for both the fast-path
check and the per-element null test. This correctly treats `NullArray`
(and any
other type without a physical validity buffer) as all-null.
- Added a sqllogictest covering the untyped-NULL case: `select
array_compact(make_array(NULL, NULL, NULL))` → `[]`.
## Are these changes tested?
Yes — new test added in
`datafusion/sqllogictest/test_files/array/array_distinct.slt` alongside
the existing `array_compact` coverage. Existing `array_compact` tests
continue to pass.
## Are there any user-facing changes?
Yes — `array_compact` on a list of untyped NULLs now returns `[]`
(matching the typed-NULL behavior and user expectation) instead of
preserving the null elements. No API changes.
---
datafusion/functions-nested/src/array_compact.rs | 33 +++++++++++++---------
.../test_files/array/array_distinct.slt | 17 +++++++++++
2 files changed, 37 insertions(+), 13 deletions(-)
diff --git a/datafusion/functions-nested/src/array_compact.rs b/datafusion/functions-nested/src/array_compact.rs
index 11be494b5b..adea9efef2 100644
--- a/datafusion/functions-nested/src/array_compact.rs
+++ b/datafusion/functions-nested/src/array_compact.rs
@@ -130,14 +130,22 @@ fn compact_list<O: OffsetSizeTrait>(
field: &Arc<arrow::datatypes::Field>,
) -> Result<ArrayRef> {
let values = list_array.values();
-
- // Fast path: no nulls in values, return input unchanged
- if values.null_count() == 0 {
+ // Use logical nulls so element types without a validity buffer
+ // (e.g. NullArray) are still treated as null.
+ let Some(values_nulls) = values.logical_nulls() else {
+ // Fast path: no validity buffer, no nulls to remove
+ return Ok(Arc::new(list_array.clone()));
+ };
+ let values_null_count = values_nulls.null_count();
+ if values_null_count == 0 {
+ // Fast path: validity buffer present but no nulls set
return Ok(Arc::new(list_array.clone()));
}
+ let list_nulls = list_array.nulls();
+ let list_offsets = list_array.offsets();
let original_data = values.to_data();
- let capacity = original_data.len() - values.null_count();
+ let capacity = original_data.len() - values_null_count;
let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
offsets.push(O::zero());
let mut mutable = MutableArrayData::with_capacities(
@@ -147,25 +155,25 @@ fn compact_list<O: OffsetSizeTrait>(
);
for row_index in 0..list_array.len() {
- if list_array.nulls().is_some_and(|n| n.is_null(row_index)) {
+ if list_nulls.is_some_and(|n| n.is_null(row_index)) {
offsets.push(offsets[row_index]);
continue;
}
- let start = list_array.offsets()[row_index].as_usize();
- let end = list_array.offsets()[row_index + 1].as_usize();
- let mut copied = 0usize;
+ let start = list_offsets[row_index].as_usize();
+ let end = list_offsets[row_index + 1].as_usize();
+ let row_null_count = values_nulls.slice(start, end - start).null_count();
+ let kept = (end - start) - row_null_count;
// Batch consecutive non-null elements into single extend() calls
// to reduce per-element overhead. For [1, 2, NULL, 3, 4] this
// produces 2 extend calls (0..2, 3..5) instead of 4 individual ones.
let mut batch_start: Option<usize> = None;
for i in start..end {
- if values.is_null(i) {
+ if values_nulls.is_null(i) {
// Null breaks the current batch — flush it
if let Some(bs) = batch_start {
mutable.extend(0, bs, i);
- copied += i - bs;
batch_start = None;
}
} else if batch_start.is_none() {
@@ -175,10 +183,9 @@ fn compact_list<O: OffsetSizeTrait>(
// Flush any remaining batch after the loop
if let Some(bs) = batch_start {
mutable.extend(0, bs, end);
- copied += end - bs;
}
- offsets.push(offsets[row_index] + O::usize_as(copied));
+ offsets.push(offsets[row_index] + O::usize_as(kept));
}
let new_values = make_array(mutable.freeze());
@@ -186,6 +193,6 @@ fn compact_list<O: OffsetSizeTrait>(
Arc::clone(field),
OffsetBuffer::new(offsets.into()),
new_values,
- list_array.nulls().cloned(),
+ list_nulls.cloned(),
)?))
}
diff --git a/datafusion/sqllogictest/test_files/array/array_distinct.slt b/datafusion/sqllogictest/test_files/array/array_distinct.slt
index 7b7033139d..777ec1ac8a 100644
--- a/datafusion/sqllogictest/test_files/array/array_distinct.slt
+++ b/datafusion/sqllogictest/test_files/array/array_distinct.slt
@@ -137,6 +137,12 @@ select array_compact(arrow_cast([NULL, NULL, NULL], 'List(Int64)'));
----
[]
+# all nulls with untyped NULLs (List(Null) inner values are a NullArray)
+query ?
+select array_compact(make_array(NULL, NULL, NULL));
+----
+[]
+
# empty array
query ?
select array_compact([]);
@@ -167,6 +173,17 @@ select array_compact([make_array(1, 2), NULL, make_array(3, 4)]);
----
[[1, 2], [3, 4]]
+# nested array of all-null inner lists: outer elements are non-null, kept as-is
+query ?
+select array_compact(make_array(make_array(NULL, NULL, NULL), make_array(NULL, NULL)));
+----
+[[NULL, NULL, NULL], [NULL, NULL]]
+
+query ?
+select array_compact(make_array(make_array(NULL, NULL, NULL), make_array(NULL, NULL, NULL)));
+----
+[[NULL, NULL, NULL], [NULL, NULL, NULL]]
+
# LargeList
query ?
select array_compact(arrow_cast([1, NULL, 2, NULL, 3], 'LargeList(Int64)'));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org