You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/10 16:07:10 UTC
(datafusion) branch main updated: perf: Optimize NULL handling in `array_slice` (#21482)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new beed4f0c1d perf: Optimize NULL handling in `array_slice` (#21482)
beed4f0c1d is described below
commit beed4f0c1d764ea675415b3d52d6b78014c7d629
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Fri Apr 10 12:05:00 2026 -0400
perf: Optimize NULL handling in `array_slice` (#21482)
## Which issue does this PR close?
- Closes #21481.
## Rationale for this change
`array_slice` does a per-row NULL checking (in four different arrays!).
It would be faster to take the union of the four input NULL buffers via
`NullBuffer::union`.
Benchmarks (Arm64):
```
- List(Int64), array args: 60.98ms -> 58.35ms (-4.3%)
- List(Int64), array args, no stride: 28.19ms -> 25.92ms (-8.0%)
- List(Int64), scalar args, no stride: 57.07ms -> 55.56ms (-2.6%)
- List(Int64), scalar args, stride=-2: 99.44ms -> 96.05ms (-3.4%)
- List(Int64), scalar args, stride=-1: 155.23ms -> 155.30ms (+0.0%)
- List(Int64), scalar args, stride=1: 58.50ms -> 55.91ms (-4.4%)
- List(Int64), scalar args, stride=2: 151.45ms -> 146.83ms (-3.1%)
- ListView(Int64), array args: 56.19ms -> 52.86ms (-5.9%)
- ListView(Int64), array args, no stride: 28.53ms -> 24.35ms (-14.7%)
- ListView(Int64), scalar args, no stride: 58.65ms -> 58.34ms (-0.5%)
- ListView(Int64), scalar args, stride=-2: 93.85ms -> 91.59ms (-2.4%)
- ListView(Int64), scalar args, stride=-1: 149.68ms -> 149.06ms (-0.4%)
- ListView(Int64), scalar args, stride=1: 59.53ms -> 58.90ms (-1.1%)
- ListView(Int64), scalar args, stride=2: 143.07ms -> 139.55ms (-2.5%)
```
## What changes are included in this PR?
## Are these changes tested?
Yes.
## Are there any user-facing changes?
No.
---
datafusion/functions-nested/src/extract.rs | 49 ++++++++++++++++--------------
1 file changed, 27 insertions(+), 22 deletions(-)
diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs
index 30b61a9bc9..4176a31f2a 100644
--- a/datafusion/functions-nested/src/extract.rs
+++ b/datafusion/functions-nested/src/extract.rs
@@ -19,9 +19,9 @@
use arrow::array::{
Array, ArrayRef, Capacities, GenericListArray, GenericListViewArray, Int64Array,
- MutableArrayData, NullArray, NullBufferBuilder, OffsetSizeTrait,
+ MutableArrayData, NullArray, OffsetSizeTrait,
};
-use arrow::buffer::{OffsetBuffer, ScalarBuffer};
+use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
use arrow::datatypes::DataType;
use arrow::datatypes::{
DataType::{FixedSizeList, LargeList, LargeListView, List, ListView, Null},
@@ -595,6 +595,23 @@ where
}
}
+/// Combine null bitmaps from all slice inputs into a single mask.
+fn combine_input_nulls(
+ array: &dyn Array,
+ from_array: &Int64Array,
+ to_array: &Int64Array,
+ stride: Option<&Int64Array>,
+) -> Option<NullBuffer> {
+ [
+ array.nulls(),
+ from_array.nulls(),
+ to_array.nulls(),
+ stride.and_then(|s| s.nulls()),
+ ]
+ .into_iter()
+ .fold(None, |acc, nulls| NullBuffer::union(acc.as_ref(), nulls))
+}
+
fn general_array_slice<O: OffsetSizeTrait>(
array: &GenericListArray<O>,
from_array: &Int64Array,
@@ -615,25 +632,19 @@ where
// The rule `adjusted_from_index` and `adjusted_to_index` follows the rule of array_slice in duckdb.
let mut offsets = vec![O::usize_as(0)];
- let mut null_builder = NullBufferBuilder::new(array.len());
+
+ let nulls = combine_input_nulls(array, from_array, to_array, stride);
for (row_index, offset_window) in array.offsets().windows(2).enumerate() {
let start = offset_window[0];
let end = offset_window[1];
let len = end - start;
- // If any input is null, return null.
- if array.is_null(row_index)
- || from_array.is_null(row_index)
- || to_array.is_null(row_index)
- || stride.is_some_and(|s| s.is_null(row_index))
- {
+ if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
mutable.extend_nulls(1);
offsets.push(offsets[row_index] + O::usize_as(1));
- null_builder.append_null();
continue;
}
- null_builder.append_non_null();
// Empty arrays always return an empty array.
if len == O::usize_as(0) {
@@ -676,7 +687,7 @@ where
Arc::new(Field::new_list_field(array.value_type(), true)),
OffsetBuffer::<O>::new(offsets.into()),
arrow::array::make_array(data),
- null_builder.finish(),
+ nulls,
)?))
}
@@ -707,21 +718,15 @@ where
let mut offsets = Vec::with_capacity(array.len());
let mut sizes = Vec::with_capacity(array.len());
let mut current_offset = O::usize_as(0);
- let mut null_builder = NullBufferBuilder::new(array.len());
+
+ let nulls = combine_input_nulls(array, from_array, to_array, stride);
for row_index in 0..array.len() {
- // Propagate NULL semantics: any NULL input yields a NULL output slot.
- if array.is_null(row_index)
- || from_array.is_null(row_index)
- || to_array.is_null(row_index)
- || stride.is_some_and(|s| s.is_null(row_index))
- {
- null_builder.append_null();
+ if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
offsets.push(current_offset);
sizes.push(O::usize_as(0));
continue;
}
- null_builder.append_non_null();
let len = array.value_size(row_index);
@@ -777,7 +782,7 @@ where
ScalarBuffer::from(offsets),
ScalarBuffer::from(sizes),
arrow::array::make_array(data),
- null_builder.finish(),
+ nulls,
)?))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org