You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/10 16:11:22 UTC
(datafusion) 01/01: perf: Optimize NULL handling in some datetime functions (#21477)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-readonly-queue/main/pr-21477-beed4f0c1d764ea675415b3d52d6b78014c7d629
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 1929d71e9962f1d6ccd5752095b794ac521db04c
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Fri Apr 10 12:10:59 2026 -0400
perf: Optimize NULL handling in some datetime functions (#21477)
## Which issue does this PR close?
- Closes #21476.
## Rationale for this change
`to_time`, `to_local_time`, and `make_date` build NULLs arrays
incrementally. It is faster and more idiomatic to build them in bulk via
`NullBuffer` instead.
This doesn't make a huge performance difference in practice, but it
shouldn't be slower, and the result is more concise and idiomatic.
Benchmarks (ARM64):
```
- make_date_col_col_col_8192: 97.0µs -> 97.3µs, +0.3%
- make_date_scalar_col_col_8192: 97.3µs -> 97.8µs, +0.5%
- make_date_scalar_scalar_col_8192: 98.1µs -> 98.7µs, +0.6%
- make_date_scalar_scalar_scalar: 162.0ns -> 162.1ns, +0.1%
- to_time_no_nulls_100k: 17.1ms -> 17.7ms, +3.5%
- to_time_10pct_nulls_100k: 16.0ms -> 16.2ms, +1.3%
- to_local_time_no_nulls_100k: 8.0ms -> 7.8ms, -2.5%
- to_local_time_10pct_nulls_100k: 7.6ms -> 7.0ms, -7.9%
```
## What changes are included in this PR?
* Add benchmarks for `to_time` and `to_local_time`
* Use 8192 batch size in `make_date` benchmark, for consistency with default DF batch size
* Implement NULL handling optimization/cleanup
## Are these changes tested?
Yes.
## Are there any user-facing changes?
No.
---
datafusion/functions/Cargo.toml | 10 +++
datafusion/functions/benches/make_date.rs | 12 +--
datafusion/functions/benches/to_local_time.rs | 90 +++++++++++++++++++++
datafusion/functions/benches/to_time.rs | 94 ++++++++++++++++++++++
datafusion/functions/src/datetime/make_time.rs | 23 ++++--
datafusion/functions/src/datetime/to_local_time.rs | 17 +---
datafusion/functions/src/datetime/to_time.rs | 14 +---
7 files changed, 223 insertions(+), 37 deletions(-)
diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml
index 6d6f26d787..967b35d2eb 100644
--- a/datafusion/functions/Cargo.toml
+++ b/datafusion/functions/Cargo.toml
@@ -186,6 +186,16 @@ harness = false
name = "to_char"
required-features = ["datetime_expressions"]
+[[bench]]
+harness = false
+name = "to_local_time"
+required-features = ["datetime_expressions"]
+
+[[bench]]
+harness = false
+name = "to_time"
+required-features = ["datetime_expressions"]
+
[[bench]]
harness = false
name = "isnan"
diff --git a/datafusion/functions/benches/make_date.rs b/datafusion/functions/benches/make_date.rs
index 42b5b10195..1c7b61ec60 100644
--- a/datafusion/functions/benches/make_date.rs
+++ b/datafusion/functions/benches/make_date.rs
@@ -30,7 +30,7 @@ use rand::rngs::ThreadRng;
fn years(rng: &mut ThreadRng) -> Int32Array {
let mut years = vec![];
- for _ in 0..1000 {
+ for _ in 0..8192 {
years.push(rng.random_range(1900..2050));
}
@@ -39,7 +39,7 @@ fn years(rng: &mut ThreadRng) -> Int32Array {
fn months(rng: &mut ThreadRng) -> Int32Array {
let mut months = vec![];
- for _ in 0..1000 {
+ for _ in 0..8192 {
months.push(rng.random_range(1..13));
}
@@ -48,14 +48,14 @@ fn months(rng: &mut ThreadRng) -> Int32Array {
fn days(rng: &mut ThreadRng) -> Int32Array {
let mut days = vec![];
- for _ in 0..1000 {
+ for _ in 0..8192 {
days.push(rng.random_range(1..29));
}
Int32Array::from(days)
}
fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("make_date_col_col_col_1000", |b| {
+ c.bench_function("make_date_col_col_col_8192", |b| {
let mut rng = rand::rng();
let years_array = Arc::new(years(&mut rng)) as ArrayRef;
let batch_len = years_array.len();
@@ -85,7 +85,7 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
- c.bench_function("make_date_scalar_col_col_1000", |b| {
+ c.bench_function("make_date_scalar_col_col_8192", |b| {
let mut rng = rand::rng();
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let months_arr = Arc::new(months(&mut rng)) as ArrayRef;
@@ -115,7 +115,7 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
- c.bench_function("make_date_scalar_scalar_col_1000", |b| {
+ c.bench_function("make_date_scalar_scalar_col_8192", |b| {
let mut rng = rand::rng();
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let month = ColumnarValue::Scalar(ScalarValue::Int32(Some(11)));
diff --git a/datafusion/functions/benches/to_local_time.rs b/datafusion/functions/benches/to_local_time.rs
new file mode 100644
index 0000000000..42d1e27198
--- /dev/null
+++ b/datafusion/functions/benches/to_local_time.rs
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::hint::black_box;
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, TimestampNanosecondArray};
+use arrow::datatypes::Field;
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+use datafusion_functions::datetime::to_local_time;
+use rand::Rng;
+use rand::rngs::ThreadRng;
+
+fn timestamps(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+ let nanos: Vec<i64> = (0..100_000)
+ .map(|_| rng.random_range(0..1_000_000_000_000_000_000i64))
+ .collect();
+ TimestampNanosecondArray::from(nanos).with_timezone("America/New_York")
+}
+
+fn timestamps_with_nulls(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+ let values: Vec<Option<i64>> = (0..100_000)
+ .map(|_| {
+ if rng.random_range(0..10u32) == 0 {
+ None
+ } else {
+ Some(rng.random_range(0..1_000_000_000_000_000_000i64))
+ }
+ })
+ .collect();
+ TimestampNanosecondArray::from(values).with_timezone("America/New_York")
+}
+
+fn bench_to_local_time(c: &mut Criterion, name: &str, array: ArrayRef) {
+ let batch_len = array.len();
+ let input = ColumnarValue::Array(array);
+ let udf = to_local_time();
+ let return_type = udf.return_type(&[input.data_type()]).unwrap();
+ let return_field = Arc::new(Field::new("f", return_type, true));
+ let arg_fields = vec![Field::new("a", input.data_type(), true).into()];
+ let config_options = Arc::new(ConfigOptions::default());
+
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: vec![input.clone()],
+ arg_fields: arg_fields.clone(),
+ number_rows: batch_len,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .expect("to_local_time should work on valid values"),
+ )
+ })
+ });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let mut rng = rand::rng();
+ bench_to_local_time(
+ c,
+ "to_local_time_no_nulls_100k",
+ Arc::new(timestamps(&mut rng)),
+ );
+ bench_to_local_time(
+ c,
+ "to_local_time_10pct_nulls_100k",
+ Arc::new(timestamps_with_nulls(&mut rng)),
+ );
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions/benches/to_time.rs b/datafusion/functions/benches/to_time.rs
new file mode 100644
index 0000000000..6b3aa19241
--- /dev/null
+++ b/datafusion/functions/benches/to_time.rs
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::hint::black_box;
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, StringArray};
+use arrow::datatypes::Field;
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+use datafusion_functions::datetime::to_time;
+use rand::Rng;
+use rand::rngs::ThreadRng;
+
+fn random_time_string(rng: &mut ThreadRng) -> String {
+ format!(
+ "{:02}:{:02}:{:02}.{:06}",
+ rng.random_range(0..24u32),
+ rng.random_range(0..60u32),
+ rng.random_range(0..60u32),
+ rng.random_range(0..1_000_000u32),
+ )
+}
+
+fn time_strings(rng: &mut ThreadRng) -> StringArray {
+ let strings: Vec<String> = (0..100_000).map(|_| random_time_string(rng)).collect();
+ StringArray::from(strings)
+}
+
+fn time_strings_with_nulls(rng: &mut ThreadRng) -> StringArray {
+ let values: Vec<Option<String>> = (0..100_000)
+ .map(|_| {
+ if rng.random_range(0..10u32) == 0 {
+ None
+ } else {
+ Some(random_time_string(rng))
+ }
+ })
+ .collect();
+ StringArray::from(values)
+}
+
+fn bench_to_time(c: &mut Criterion, name: &str, array: ArrayRef) {
+ let batch_len = array.len();
+ let input = ColumnarValue::Array(array);
+ let udf = to_time();
+ let return_type = udf.return_type(&[input.data_type()]).unwrap();
+ let return_field = Arc::new(Field::new("f", return_type, true));
+ let arg_fields = vec![Field::new("a", input.data_type(), true).into()];
+ let config_options = Arc::new(ConfigOptions::default());
+
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: vec![input.clone()],
+ arg_fields: arg_fields.clone(),
+ number_rows: batch_len,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .expect("to_time should work on valid values"),
+ )
+ })
+ });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let mut rng = rand::rng();
+ bench_to_time(c, "to_time_no_nulls_100k", Arc::new(time_strings(&mut rng)));
+ bench_to_time(
+ c,
+ "to_time_10pct_nulls_100k",
+ Arc::new(time_strings_with_nulls(&mut rng)),
+ );
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions/src/datetime/make_time.rs b/datafusion/functions/src/datetime/make_time.rs
index 690acafa8d..718612fcad 100644
--- a/datafusion/functions/src/datetime/make_time.rs
+++ b/datafusion/functions/src/datetime/make_time.rs
@@ -17,10 +17,10 @@
use std::sync::Arc;
-use arrow::array::builder::PrimitiveBuilder;
use arrow::array::cast::AsArray;
use arrow::array::types::Int32Type;
use arrow::array::{Array, PrimitiveArray};
+use arrow::buffer::NullBuffer;
use arrow::datatypes::DataType::Time32;
use arrow::datatypes::{DataType, Time32SecondType, TimeUnit};
use chrono::prelude::*;
@@ -142,24 +142,31 @@ impl ScalarUDFImpl for MakeTimeFunc {
let minutes = minutes.as_primitive::<Int32Type>();
let seconds = seconds.as_primitive::<Int32Type>();
- let mut builder: PrimitiveBuilder<Time32SecondType> =
- PrimitiveArray::builder(len);
+ let nulls = NullBuffer::union(
+ NullBuffer::union(hours.nulls(), minutes.nulls()).as_ref(),
+ seconds.nulls(),
+ );
+ let mut values = Vec::with_capacity(len);
for i in 0..len {
- // match postgresql behaviour which returns null for any null input
- if hours.is_null(i) || minutes.is_null(i) || seconds.is_null(i) {
- builder.append_null();
+ // Match Postgres behaviour which returns null for any null input
+ if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
+ values.push(0);
} else {
make_time_inner(
hours.value(i),
minutes.value(i),
seconds.value(i),
- |seconds: i32| builder.append_value(seconds),
+ |seconds: i32| values.push(seconds),
)?;
}
}
- Ok(ColumnarValue::Array(Arc::new(builder.finish())))
+ Ok(ColumnarValue::Array(Arc::new(PrimitiveArray::<
+ Time32SecondType,
+ >::new(
+ values.into(), nulls
+ ))))
}
}
}
diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rs
index f0d489d1b1..5bd1978893 100644
--- a/datafusion/functions/src/datetime/to_local_time.rs
+++ b/datafusion/functions/src/datetime/to_local_time.rs
@@ -19,7 +19,7 @@ use std::ops::Add;
use std::sync::Arc;
use arrow::array::timezone::Tz;
-use arrow::array::{ArrayRef, PrimitiveBuilder};
+use arrow::array::{ArrayRef, PrimitiveArray};
use arrow::datatypes::DataType::Timestamp;
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
@@ -153,18 +153,9 @@ fn transform_array<T: ArrowTimestampType>(
tz: Tz,
) -> Result<ColumnarValue> {
let primitive_array = as_primitive_array::<T>(array)?;
- let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len());
- for ts_opt in primitive_array.iter() {
- match ts_opt {
- None => builder.append_null(),
- Some(ts) => {
- let adjusted_ts: i64 = adjust_to_local_time::<T>(ts, tz)?;
- builder.append_value(adjusted_ts)
- }
- }
- }
-
- Ok(ColumnarValue::Array(Arc::new(builder.finish())))
+ let result: PrimitiveArray<T> =
+ primitive_array.try_unary(|ts| adjust_to_local_time::<T>(ts, tz))?;
+ Ok(ColumnarValue::Array(Arc::new(result)))
}
fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue> {
diff --git a/datafusion/functions/src/datetime/to_time.rs b/datafusion/functions/src/datetime/to_time.rs
index 815bf6a328..94aa49fbba 100644
--- a/datafusion/functions/src/datetime/to_time.rs
+++ b/datafusion/functions/src/datetime/to_time.rs
@@ -16,7 +16,6 @@
// under the License.
use crate::datetime::common::*;
-use arrow::array::builder::PrimitiveBuilder;
use arrow::array::cast::AsArray;
use arrow::array::temporal_conversions::time_to_time64ns;
use arrow::array::types::Time64NanosecondType;
@@ -213,20 +212,15 @@ fn parse_time_array<'a, A: StringArrayType<'a>>(
array: &A,
formats: &[&str],
) -> Result<PrimitiveArray<Time64NanosecondType>> {
- let mut builder: PrimitiveBuilder<Time64NanosecondType> =
- PrimitiveArray::builder(array.len());
-
+ let mut values = Vec::with_capacity(array.len());
for i in 0..array.len() {
if array.is_null(i) {
- builder.append_null();
+ values.push(0);
} else {
- let s = array.value(i);
- let nanos = parse_time_with_formats(s, formats)?;
- builder.append_value(nanos);
+ values.push(parse_time_with_formats(array.value(i), formats)?);
}
}
-
- Ok(builder.finish())
+ Ok(PrimitiveArray::new(values.into(), array.nulls().cloned()))
}
/// Parse time string using provided formats
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org