You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/19 12:02:45 UTC
(datafusion) branch gh-readonly-queue/main/pr-21544-d8c9797296dd69655b476955621c8e32a96f898a created (now 43d32a8c14)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-21544-d8c9797296dd69655b476955621c8e32a96f898a
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 43d32a8c14 chore: use bench array helpers from Arrow bench_util (#21544)
This branch includes the following new commits:
new 43d32a8c14 chore: use bench array helpers from Arrow bench_util (#21544)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: chore: use bench array helpers from Arrow bench_util (#21544)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-21544-d8c9797296dd69655b476955621c8e32a96f898a
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 43d32a8c14b12f388e04719170a7704f84b973b6
Author: theirix <th...@gmail.com>
AuthorDate: Sun Apr 19 13:02:23 2026 +0100
chore: use bench array helpers from Arrow bench_util (#21544)
## Which issue does this PR close?
- Closes #.
## Rationale for this change
While working on function benchmarks, I've noticed duplicated code,
which can be replaced with
[arrow::util::bench_util](https://github.com/apache/arrow-rs/blob/main/arrow/src/util/bench_util.rs)
helpers. I recall a discussion in
https://github.com/apache/arrow-rs/issues/7294 about bench_util /
test_utils usage, and seems like it can be used to simplify clients
code.
## What changes are included in this PR?
- Equivalent changes in benchmarks. More complicated and targeted
helpers are left as is
## Are these changes tested?
- Run benchmarks to check errors or performance degradations
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---------
Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
.../functions-aggregate/benches/array_agg.rs | 20 +--
datafusion/functions-nested/Cargo.toml | 1 +
.../functions-nested/benches/array_min_max.rs | 69 ++--------
.../functions-nested/benches/array_remove.rs | 133 +++++--------------
.../functions-nested/benches/array_repeat.rs | 143 ++++++---------------
.../functions-nested/benches/array_to_string.rs | 65 +++-------
6 files changed, 100 insertions(+), 331 deletions(-)
diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs
index 793c2aac96..b0d8148c3e 100644
--- a/datafusion/functions-aggregate/benches/array_agg.rs
+++ b/datafusion/functions-aggregate/benches/array_agg.rs
@@ -20,7 +20,6 @@ use std::sync::Arc;
use arrow::array::{
Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder,
- PrimitiveArray,
};
use arrow::datatypes::{Field, Int64Type};
use criterion::{Criterion, criterion_group, criterion_main};
@@ -28,6 +27,7 @@ use datafusion_expr::Accumulator;
use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator;
use arrow::buffer::OffsetBuffer;
+use arrow::util::bench_util::create_primitive_array;
use rand::Rng;
use rand::SeedableRng;
use rand::distr::{Distribution, StandardUniform};
@@ -54,24 +54,6 @@ fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) {
});
}
-pub fn create_primitive_array<T>(size: usize, null_density: f32) -> PrimitiveArray<T>
-where
- T: ArrowPrimitiveType,
- StandardUniform: Distribution<T::Native>,
-{
- let mut rng = seedable_rng();
-
- (0..size)
- .map(|_| {
- if rng.random::<f32>() < null_density {
- None
- } else {
- Some(rng.random())
- }
- })
- .collect()
-}
-
/// Create List array with the given item data type, null density, null locations and zero length lists density
/// Creates a random (but fixed-seeded) array of a given size and null density
pub fn create_list_array<T>(
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index 13b0029615..d2204a7be7 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -64,6 +64,7 @@ log = { workspace = true }
memchr = { workspace = true }
[dev-dependencies]
+arrow = { workspace = true, features = ["test_utils"] }
criterion = { workspace = true, features = ["async_tokio"] }
rand = { workspace = true }
diff --git a/datafusion/functions-nested/benches/array_min_max.rs b/datafusion/functions-nested/benches/array_min_max.rs
index 391a84d148..45838da79f 100644
--- a/datafusion/functions-nested/benches/array_min_max.rs
+++ b/datafusion/functions-nested/benches/array_min_max.rs
@@ -17,73 +17,19 @@
use std::sync::Arc;
-use arrow::array::{ArrayRef, Int64Array, ListArray};
-use arrow::buffer::{NullBuffer, OffsetBuffer};
-use arrow::datatypes::{DataType, Field};
+use arrow::array::{Array, ArrayRef};
+use arrow::datatypes::{DataType, Field, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
use datafusion_functions_nested::min_max::ArrayMax;
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
const NUM_ROWS: usize = 8192;
const SEED: u64 = 42;
const LIST_NULL_DENSITY: f64 = 0.1;
const ELEMENT_NULL_DENSITY: f64 = 0.1;
-fn create_int64_list_array(
- num_rows: usize,
- list_size: usize,
- element_null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let total_values = num_rows * list_size;
-
- if element_null_density > 0.0 {
- let values: Vec<Option<i64>> = (0..total_values)
- .map(|_| {
- if rng.random::<f64>() < element_null_density {
- None
- } else {
- Some(rng.random::<i64>() % 10_000)
- }
- })
- .collect();
- let values_array = Arc::new(Int64Array::from(values));
-
- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();
- let nulls: Vec<bool> = (0..num_rows)
- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)
- .collect();
-
- Arc::new(ListArray::new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- values_array,
- Some(NullBuffer::from(nulls)),
- ))
- } else {
- // No element nulls — values array has no null buffer
- let values: Vec<i64> = (0..total_values)
- .map(|_| rng.random::<i64>() % 10_000)
- .collect();
- let values_array = Arc::new(Int64Array::from(values));
-
- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();
- let nulls: Vec<bool> = (0..num_rows)
- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)
- .collect();
-
- Arc::new(ListArray::new(
- Arc::new(Field::new("item", DataType::Int64, false)),
- OffsetBuffer::new(offsets.into()),
- values_array,
- Some(NullBuffer::from(nulls)),
- ))
- }
-}
-
fn criterion_benchmark(c: &mut Criterion) {
let udf = ArrayMax::new();
let config_options = Arc::new(ConfigOptions::default());
@@ -91,7 +37,14 @@ fn criterion_benchmark(c: &mut Criterion) {
for list_size in [10, 100, 1000] {
for (label, null_density) in [("nulls", ELEMENT_NULL_DENSITY), ("no_nulls", 0.0)]
{
- let list_array = create_int64_list_array(NUM_ROWS, list_size, null_density);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ LIST_NULL_DENSITY as f32,
+ null_density as f32,
+ list_size,
+ SEED,
+ ));
let args = vec![ColumnarValue::Array(Arc::clone(&list_array))];
let arg_fields =
vec![Field::new("arg_0", list_array.data_type().clone(), true).into()];
diff --git a/datafusion/functions-nested/benches/array_remove.rs b/datafusion/functions-nested/benches/array_remove.rs
index 6fd153548a..3384bd0e46 100644
--- a/datafusion/functions-nested/benches/array_remove.rs
+++ b/datafusion/functions-nested/benches/array_remove.rs
@@ -16,11 +16,12 @@
// under the License.
use arrow::array::{
- Array, ArrayRef, BinaryArray, BooleanArray, Decimal128Array, FixedSizeBinaryArray,
- Float64Array, Int64Array, ListArray, StringArray,
+ Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeBinaryArray, ListArray,
+ StringArray,
};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Decimal128Type, Field, Float64Type, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{
criterion_group, criterion_main, {BenchmarkId, Criterion},
};
@@ -55,7 +56,15 @@ fn bench_array_remove_int64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_int64");
for &array_size in ARRAY_SIZES {
- let list_array = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
+
let element_to_remove = ScalarValue::Int64(Some(1));
let args = create_args(list_array.clone(), element_to_remove.clone());
@@ -96,7 +105,14 @@ fn bench_array_remove_f64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_f64");
for &array_size in ARRAY_SIZES {
- let list_array = create_f64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
let element_to_remove = ScalarValue::Float64(Some(1.0));
let args = create_args(list_array.clone(), element_to_remove.clone());
@@ -260,8 +276,17 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_decimal64");
for &array_size in ARRAY_SIZES {
- let list_array = create_decimal64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
- let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 10, 2);
+ let list_array: ArrayRef = Arc::new(create_primitive_list_array_with_seed::<
+ i32,
+ Decimal128Type,
+ >(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
+ let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 38, 10);
let args = create_args(list_array.clone(), element_to_remove.clone());
group.bench_with_input(
@@ -276,7 +301,7 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {
arg_fields: vec![
Field::new("arr", list_array.data_type().clone(), false)
.into(),
- Field::new("el", DataType::Decimal128(10, 2), false)
+ Field::new("el", DataType::Decimal128(38, 10), false)
.into(),
],
number_rows: NUM_ROWS,
@@ -348,66 +373,6 @@ fn create_args(list_array: ArrayRef, element: ScalarValue) -> Vec<ColumnarValue>
]
}
-fn create_int64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size as i64))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
-fn create_f64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size as i64) as f64)
- }
- })
- .collect::<Float64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Float64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
fn create_string_list_array(
num_rows: usize,
array_size: usize,
@@ -500,38 +465,6 @@ fn create_boolean_list_array(
)
}
-fn create_decimal64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size) as i128 * 100)
- }
- })
- .collect::<Decimal128Array>()
- .with_precision_and_scale(10, 2)
- .unwrap();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Decimal128(10, 2), true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
fn create_fixed_size_binary_list_array(
num_rows: usize,
array_size: usize,
diff --git a/datafusion/functions-nested/benches/array_repeat.rs b/datafusion/functions-nested/benches/array_repeat.rs
index 0ce8db00ce..42372322e2 100644
--- a/datafusion/functions-nested/benches/array_repeat.rs
+++ b/datafusion/functions-nested/benches/array_repeat.rs
@@ -15,9 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int64Array, ListArray};
+use arrow::array::{ArrayRef, ListArray};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, Int64Type};
+use arrow::util::bench_util::{
+ create_boolean_array, create_f64_array, create_primitive_array,
+ create_primitive_list_array_with_seed, create_string_array_with_max_len,
+};
use criterion::{
criterion_group, criterion_main, {BenchmarkId, Criterion},
};
@@ -32,7 +36,8 @@ use std::hint::black_box;
use std::sync::Arc;
const NUM_ROWS: &[usize] = &[100, 1000, 10000];
-const REPEAT_COUNTS: &[u64] = &[5, 50];
+// Must be of type i64 because ArrayRepeat's second argument is Int64
+const REPEAT_COUNTS: &[i64] = &[5, 50];
const SEED: u64 = 42;
const NULL_DENSITY: f64 = 0.1;
@@ -52,7 +57,10 @@ fn bench_array_repeat_int64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_int64");
for &num_rows in NUM_ROWS {
- let element_array = create_int64_array(num_rows, NULL_DENSITY);
+ let element_array: ArrayRef = Arc::new(create_primitive_array::<Int64Type>(
+ num_rows,
+ NULL_DENSITY as f32,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -71,7 +79,7 @@ fn bench_array_repeat_int64(c: &mut Criterion) {
args: args.clone(),
arg_fields: vec![
Field::new("element", DataType::Int64, false).into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -100,7 +108,11 @@ fn bench_array_repeat_string(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_string");
for &num_rows in NUM_ROWS {
- let element_array = create_string_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_string_array_with_max_len::<i64>(
+ num_rows,
+ NULL_DENSITY as f32,
+ 100,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -119,7 +131,7 @@ fn bench_array_repeat_string(c: &mut Criterion) {
args: args.clone(),
arg_fields: vec![
Field::new("element", DataType::Utf8, false).into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -148,7 +160,14 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_nested_int64");
for &num_rows in NUM_ROWS {
- let list_array = create_int64_list_array(num_rows, 5, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ num_rows,
+ NULL_DENSITY as f32,
+ NULL_DENSITY as f32,
+ 5,
+ SEED,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -172,7 +191,7 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {
false,
)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -201,7 +220,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_float64");
for &num_rows in NUM_ROWS {
- let element_array = create_float64_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_f64_array(num_rows, NULL_DENSITY as f32));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -221,7 +240,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {
arg_fields: vec![
Field::new("element", DataType::Float64, false)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -250,7 +269,11 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_boolean");
for &num_rows in NUM_ROWS {
- let element_array = create_boolean_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_boolean_array(
+ num_rows,
+ NULL_DENSITY as f32,
+ f32::MAX,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -270,7 +293,7 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {
arg_fields: vec![
Field::new("element", DataType::Boolean, false)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -323,7 +346,7 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {
false,
)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -348,98 +371,6 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {
group.finish();
}
-fn create_int64_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
-
- Arc::new(values)
-}
-
-fn create_string_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- use arrow::array::StringArray;
-
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(format!("value_{}", rng.random_range(0..100)))
- }
- })
- .collect::<StringArray>();
-
- Arc::new(values)
-}
-
-fn create_int64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
-fn create_float64_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0.0..1000.0))
- }
- })
- .collect::<Float64Array>();
-
- Arc::new(values)
-}
-
-fn create_boolean_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random())
- }
- })
- .collect::<BooleanArray>();
-
- Arc::new(values)
-}
-
fn create_string_list_array(
num_rows: usize,
array_size: usize,
diff --git a/datafusion/functions-nested/benches/array_to_string.rs b/datafusion/functions-nested/benches/array_to_string.rs
index 286ed4eeb0..4b63d70548 100644
--- a/datafusion/functions-nested/benches/array_to_string.rs
+++ b/datafusion/functions-nested/benches/array_to_string.rs
@@ -15,9 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, Float64Array, Int64Array, ListArray, StringArray};
+use arrow::array::{Array, ArrayRef, ListArray, StringArray};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, Float64Type, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -89,29 +90,13 @@ fn bench_array_to_string(
}
fn create_int64_list_array(array_size: usize) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..NUM_ROWS * array_size)
- .map(|_| {
- if rng.random::<f64>() < NULL_DENSITY {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=NUM_ROWS)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ))
}
fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {
@@ -133,29 +118,13 @@ fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {
}
fn create_float64_list_array(array_size: usize) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..NUM_ROWS * array_size)
- .map(|_| {
- if rng.random::<f64>() < NULL_DENSITY {
- None
- } else {
- Some(rng.random_range(-1000.0..1000.0))
- }
- })
- .collect::<Float64Array>();
- let offsets = (0..=NUM_ROWS)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Float64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ))
}
fn create_string_list_array(array_size: usize) -> ArrayRef {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org