You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/10 16:07:10 UTC

(datafusion) branch main updated: perf: Optimize NULL handling in `array_slice` (#21482)

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new beed4f0c1d perf: Optimize NULL handling in `array_slice` (#21482)
beed4f0c1d is described below

commit beed4f0c1d764ea675415b3d52d6b78014c7d629
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Fri Apr 10 12:05:00 2026 -0400

    perf: Optimize NULL handling in `array_slice` (#21482)
    
    ## Which issue does this PR close?
    
    - Closes #21481.
    
    ## Rationale for this change
    
    `array_slice` does a per-row NULL checking (in four different arrays!).
    It would be faster to take the union of the four input NULL buffers via
    `NullBuffer::union`.
    
    Benchmarks (Arm64):
    ```
      - List(Int64), array args:                  60.98ms -> 58.35ms  (-4.3%)
      - List(Int64), array args, no stride:       28.19ms -> 25.92ms  (-8.0%)
      - List(Int64), scalar args, no stride:      57.07ms -> 55.56ms  (-2.6%)
      - List(Int64), scalar args, stride=-2:      99.44ms -> 96.05ms  (-3.4%)
      - List(Int64), scalar args, stride=-1:     155.23ms -> 155.30ms (+0.0%)
      - List(Int64), scalar args, stride=1:       58.50ms -> 55.91ms  (-4.4%)
      - List(Int64), scalar args, stride=2:      151.45ms -> 146.83ms (-3.1%)
      - ListView(Int64), array args:              56.19ms -> 52.86ms  (-5.9%)
      - ListView(Int64), array args, no stride:   28.53ms -> 24.35ms  (-14.7%)
      - ListView(Int64), scalar args, no stride:  58.65ms -> 58.34ms  (-0.5%)
      - ListView(Int64), scalar args, stride=-2:  93.85ms -> 91.59ms  (-2.4%)
      - ListView(Int64), scalar args, stride=-1: 149.68ms -> 149.06ms (-0.4%)
      - ListView(Int64), scalar args, stride=1:   59.53ms -> 58.90ms  (-1.1%)
      - ListView(Int64), scalar args, stride=2:  143.07ms -> 139.55ms (-2.5%)
    ```
    
    ## What changes are included in this PR?
    
    ## Are these changes tested?
    
    Yes.
    
    ## Are there any user-facing changes?
    
    No.
---
 datafusion/functions-nested/src/extract.rs | 49 ++++++++++++++++--------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs
index 30b61a9bc9..4176a31f2a 100644
--- a/datafusion/functions-nested/src/extract.rs
+++ b/datafusion/functions-nested/src/extract.rs
@@ -19,9 +19,9 @@
 
 use arrow::array::{
     Array, ArrayRef, Capacities, GenericListArray, GenericListViewArray, Int64Array,
-    MutableArrayData, NullArray, NullBufferBuilder, OffsetSizeTrait,
+    MutableArrayData, NullArray, OffsetSizeTrait,
 };
-use arrow::buffer::{OffsetBuffer, ScalarBuffer};
+use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 use arrow::datatypes::DataType;
 use arrow::datatypes::{
     DataType::{FixedSizeList, LargeList, LargeListView, List, ListView, Null},
@@ -595,6 +595,23 @@ where
     }
 }
 
+/// Combine null bitmaps from all slice inputs into a single mask.
+fn combine_input_nulls(
+    array: &dyn Array,
+    from_array: &Int64Array,
+    to_array: &Int64Array,
+    stride: Option<&Int64Array>,
+) -> Option<NullBuffer> {
+    [
+        array.nulls(),
+        from_array.nulls(),
+        to_array.nulls(),
+        stride.and_then(|s| s.nulls()),
+    ]
+    .into_iter()
+    .fold(None, |acc, nulls| NullBuffer::union(acc.as_ref(), nulls))
+}
+
 fn general_array_slice<O: OffsetSizeTrait>(
     array: &GenericListArray<O>,
     from_array: &Int64Array,
@@ -615,25 +632,19 @@ where
     // The rule `adjusted_from_index` and `adjusted_to_index` follows the rule of array_slice in duckdb.
 
     let mut offsets = vec![O::usize_as(0)];
-    let mut null_builder = NullBufferBuilder::new(array.len());
+
+    let nulls = combine_input_nulls(array, from_array, to_array, stride);
 
     for (row_index, offset_window) in array.offsets().windows(2).enumerate() {
         let start = offset_window[0];
         let end = offset_window[1];
         let len = end - start;
 
-        // If any input is null, return null.
-        if array.is_null(row_index)
-            || from_array.is_null(row_index)
-            || to_array.is_null(row_index)
-            || stride.is_some_and(|s| s.is_null(row_index))
-        {
+        if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
             mutable.extend_nulls(1);
             offsets.push(offsets[row_index] + O::usize_as(1));
-            null_builder.append_null();
             continue;
         }
-        null_builder.append_non_null();
 
         // Empty arrays always return an empty array.
         if len == O::usize_as(0) {
@@ -676,7 +687,7 @@ where
         Arc::new(Field::new_list_field(array.value_type(), true)),
         OffsetBuffer::<O>::new(offsets.into()),
         arrow::array::make_array(data),
-        null_builder.finish(),
+        nulls,
     )?))
 }
 
@@ -707,21 +718,15 @@ where
     let mut offsets = Vec::with_capacity(array.len());
     let mut sizes = Vec::with_capacity(array.len());
     let mut current_offset = O::usize_as(0);
-    let mut null_builder = NullBufferBuilder::new(array.len());
+
+    let nulls = combine_input_nulls(array, from_array, to_array, stride);
 
     for row_index in 0..array.len() {
-        // Propagate NULL semantics: any NULL input yields a NULL output slot.
-        if array.is_null(row_index)
-            || from_array.is_null(row_index)
-            || to_array.is_null(row_index)
-            || stride.is_some_and(|s| s.is_null(row_index))
-        {
-            null_builder.append_null();
+        if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
             offsets.push(current_offset);
             sizes.push(O::usize_as(0));
             continue;
         }
-        null_builder.append_non_null();
 
         let len = array.value_size(row_index);
 
@@ -777,7 +782,7 @@ where
         ScalarBuffer::from(offsets),
         ScalarBuffer::from(sizes),
         arrow::array::make_array(data),
-        null_builder.finish(),
+        nulls,
     )?))
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org