(datafusion) branch main updated: perf: reuse mask in `truncate_list_nulls` and avoid counting all true bits (#22158)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 18a219c728 perf: reuse mask in `truncate_list_nulls` and avoid counting all true bits (#22158)
18a219c728 is described below

commit 18a219c7285278d90c91d6ed51ee96a43e6a35e6
Author: Raz Luvaton <16...@users.noreply.github.com>
AuthorDate: Thu May 14 16:21:17 2026 +0300

    perf: reuse mask in `truncate_list_nulls` and avoid counting all true bits (#22158)
    
    ## Which issue does this PR close?
    
    N/A
    
    ## Rationale for this change
    
    1. instead of counting set bits to check if there is at least 1 set
    bits, we can use the existing helpers on `BooleanArray` that check if
    there is at least 1 set bit
    2. Avoid unnecessary `BooleanBuffer` bitwise operations and reuse mask
    
    ## What changes are included in this PR?
    
    reused mask, and use helper to check if at least one false
    
    ## Are these changes tested?
    
    Existing tests
    
    ## Are there any user-facing changes?
    
    No
    
    ------
    
    Cc @gstvg, @comphead
---
 datafusion/common/src/utils/mod.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index acee7b7a84..0c667b17c3 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -31,12 +31,12 @@ use arrow::array::{
     cast::AsArray,
 };
 use arrow::array::{
-    ArrowPrimitiveType, Datum, GenericListArray, Int32Array, Int64Array,
+    ArrowPrimitiveType, BooleanArray, Datum, GenericListArray, Int32Array, Int64Array,
     MutableArrayData, PrimitiveArray, make_array,
 };
 use arrow::array::{LargeListViewArray, ListViewArray};
 use arrow::buffer::{OffsetBuffer, ScalarBuffer};
-use arrow::compute::kernels::cmp::neq;
+use arrow::compute::kernels::cmp::eq;
 use arrow::compute::kernels::length::length;
 use arrow::compute::{SortColumn, SortOptions, partition};
 use arrow::datatypes::{
@@ -1129,6 +1129,7 @@ pub fn remove_list_null_values(array: &ArrayRef) -> Result<ArrayRef> {
     }
 }
 
+/// Create a new list array where all the nulls point to empty lists
 fn truncate_list_nulls<O: OffsetSizeTrait>(
     list: &GenericListArray<O>,
 ) -> Result<GenericListArray<O>> {
@@ -1142,17 +1143,18 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(
             &Int64Array::new_scalar(0)
         };
 
-        let not_empty = neq(&lengths, zero)?;
-        let null_and_non_empty = &!nulls.inner() & not_empty.values();
+        let (mut valid_or_empty, _nulls) = eq(&lengths, zero)?.into_parts();
+        valid_or_empty |= nulls.inner();
+        let valid_or_empty = BooleanArray::from(valid_or_empty);
 
-        if null_and_non_empty.count_set_bits() > 0 {
+        if valid_or_empty.has_false() {
             let array_data = list.values().to_data();
             let offsets = list.offsets();
             let capacity = offsets[offsets.len() - 1] - offsets[0];
             let mut mutable_array_data =
                 MutableArrayData::new(vec![&array_data], false, capacity.as_usize());
 
-            let valid_or_empty = nulls.inner() | &!not_empty.values();
+            let (valid_or_empty, _nulls) = valid_or_empty.into_parts();
 
             for (start, end) in valid_or_empty.set_slices() {
                 mutable_array_data.extend(


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org