You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/31 21:12:23 UTC
(datafusion) branch gh-readonly-queue/main/pr-23653-dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48 created (now 2eaba476e7)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23653-dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 2eaba476e7 bench: use seedable rng for reproducibility (#23653)
This branch includes the following new commits:
new 2eaba476e7 bench: use seedable rng for reproducibility (#23653)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: bench: use seedable rng for reproducibility (#23653)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23653-dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 2eaba476e7678325c1d61becabee91d1b1020b22
Author: theirix <th...@gmail.com>
AuthorDate: Fri Jul 31 22:12:00 2026 +0100
bench: use seedable rng for reproducibility (#23653)
## Which issue does this PR close?
- Closes #23652.
## Rationale for this change
It's a follow-up PR to the [noisy
benchmark](https://github.com/apache/datafusion/pull/23586#issuecomment-4984137575)
issue. Let's use a seeded StdRng everywhere in benchmarks to make
results more predictable. It is widely used now.
## What changes are included in this PR?
Switch from a randomly initialised generator to a seeded generator
## Are these changes tested?
A recetnt `cargo bench --bench pad` runs fine
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---
benchmarks/src/cancellation.rs | 16 +++++-----
datafusion/core/benches/map_query_sql.rs | 9 +++---
datafusion/core/benches/parquet_query_sql.rs | 36 +++++++++++-----------
datafusion/core/benches/parquet_struct_query.rs | 11 ++++---
datafusion/functions-nested/benches/map.rs | 18 +++++------
datafusion/functions/benches/concat.rs | 18 ++++++-----
datafusion/functions/benches/concat_ws.rs | 10 +++----
datafusion/functions/benches/date_bin.rs | 7 ++---
datafusion/functions/benches/gcd.rs | 4 +--
datafusion/functions/benches/lcm.rs | 4 +--
datafusion/functions/benches/make_date.rs | 15 +++++-----
datafusion/functions/benches/pad.rs | 10 +++----
datafusion/functions/benches/regx.rs | 36 +++++++++++-----------
datafusion/functions/benches/to_char.rs | 40 ++++++++++++-------------
datafusion/functions/benches/to_local_time.rs | 9 +++---
datafusion/functions/benches/to_time.rs | 11 ++++---
16 files changed, 122 insertions(+), 132 deletions(-)
diff --git a/benchmarks/src/cancellation.rs b/benchmarks/src/cancellation.rs
index d3da1b0e83..1048fa0989 100644
--- a/benchmarks/src/cancellation.rs
+++ b/benchmarks/src/cancellation.rs
@@ -39,9 +39,8 @@ use futures::TryStreamExt;
use object_store::ObjectStore;
use parquet::arrow::AsyncArrowWriter;
use parquet::arrow::async_writer::ParquetObjectWriter;
-use rand::Rng;
use rand::distr::Alphanumeric;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
use tokio::runtime::Runtime;
use tokio_util::sync::CancellationToken;
@@ -215,7 +214,8 @@ async fn find_or_generate_files(
if files_on_disk.is_empty() {
println!("No data files found, generating (this will take a bit)");
- generate_data(data_dir.as_ref(), num_files, num_rows_per_file).await?;
+ let mut rng = StdRng::seed_from_u64(0);
+ generate_data(&mut rng, data_dir.as_ref(), num_files, num_rows_per_file).await?;
println!("Done generating files");
let files_on_disk = find_files_on_disk(data_dir)?;
@@ -269,6 +269,7 @@ async fn load_data(
}
async fn generate_data(
+ rng: &mut StdRng,
data_dir: impl AsRef<Path>,
num_files: usize,
num_rows_per_file: usize,
@@ -295,7 +296,7 @@ async fn generate_data(
for file_num in 1..=num_files {
println!("Generating file {file_num} of {num_files}");
let data = columns.iter().map(|(column_name, column_type)| {
- let column = random_data(column_type, num_rows_per_file);
+ let column = random_data(rng, column_type, num_rows_per_file);
(column_name, column)
});
let to_write = RecordBatch::try_from_iter(data).unwrap();
@@ -311,13 +312,12 @@ async fn generate_data(
Ok(())
}
-fn random_data(column_type: &DataType, rows: usize) -> Arc<dyn Array> {
- let mut rng = rand::rng();
- let values = (0..rows).map(|_| random_value(&mut rng, column_type));
+fn random_data(rng: &mut StdRng, column_type: &DataType, rows: usize) -> Arc<dyn Array> {
+ let values = (0..rows).map(|_| random_value(rng, column_type));
ScalarValue::iter_to_array(values).unwrap()
}
-fn random_value(rng: &mut ThreadRng, column_type: &DataType) -> ScalarValue {
+fn random_value(rng: &mut StdRng, column_type: &DataType) -> ScalarValue {
match column_type {
DataType::Float64 => ScalarValue::Float64(Some(rng.random())),
DataType::Boolean => ScalarValue::Boolean(Some(rng.random())),
diff --git a/datafusion/core/benches/map_query_sql.rs b/datafusion/core/benches/map_query_sql.rs
index 67904197bc..6e7d584c6f 100644
--- a/datafusion/core/benches/map_query_sql.rs
+++ b/datafusion/core/benches/map_query_sql.rs
@@ -22,8 +22,7 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
use criterion::{Criterion, criterion_group, criterion_main};
use parking_lot::Mutex;
-use rand::Rng;
-use rand::prelude::ThreadRng;
+use rand::prelude::*;
use tokio::runtime::Runtime;
use datafusion::prelude::SessionContext;
@@ -33,7 +32,7 @@ use datafusion_functions_nested::map::map;
mod data_utils;
-fn build_keys(rng: &mut ThreadRng) -> Vec<String> {
+fn build_keys(rng: &mut StdRng) -> Vec<String> {
let mut keys = HashSet::with_capacity(1000);
while keys.len() < 1000 {
let key = rng.random_range(0..9999).to_string();
@@ -42,7 +41,7 @@ fn build_keys(rng: &mut ThreadRng) -> Vec<String> {
keys.into_iter().collect()
}
-fn build_values(rng: &mut ThreadRng) -> Vec<i32> {
+fn build_values(rng: &mut StdRng) -> Vec<i32> {
let mut values = vec![];
for _ in 0..1000 {
values.push(rng.random_range(0..9999));
@@ -67,7 +66,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let df = rt.block_on(ctx.lock().table("t")).unwrap();
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let keys = build_keys(&mut rng);
let values = build_values(&mut rng);
let mut key_buffer = Vec::new();
diff --git a/datafusion/core/benches/parquet_query_sql.rs b/datafusion/core/benches/parquet_query_sql.rs
index f099137973..2e7794bfd1 100644
--- a/datafusion/core/benches/parquet_query_sql.rs
+++ b/datafusion/core/benches/parquet_query_sql.rs
@@ -32,7 +32,6 @@ use parquet::file::properties::{WriterProperties, WriterVersion};
use rand::distr::Alphanumeric;
use rand::distr::uniform::SampleUniform;
use rand::prelude::*;
-use rand::rng;
use std::fs::File;
use std::io::Read;
use std::ops::Range;
@@ -69,36 +68,36 @@ fn schema() -> SchemaRef {
]))
}
-fn generate_batch() -> RecordBatch {
+fn generate_batch(rng: &mut StdRng) -> RecordBatch {
let schema = schema();
let len = WRITE_RECORD_BATCH_SIZE;
RecordBatch::try_new(
schema,
vec![
- generate_string_dictionary("prefix", 10, len, 1.0),
- generate_string_dictionary("prefix", 10, len, 0.5),
- generate_string_dictionary("prefix", 100, len, 1.0),
- generate_string_dictionary("prefix", 100, len, 0.5),
- generate_string_dictionary("prefix", 1000, len, 1.0),
- generate_string_dictionary("prefix", 1000, len, 0.5),
- generate_strings(0..100, len, 1.0),
- generate_strings(0..100, len, 0.5),
- generate_primitive::<Int64Type>(len, 1.0, -2000..2000),
- generate_primitive::<Int64Type>(len, 0.5, -2000..2000),
- generate_primitive::<Float64Type>(len, 1.0, -1000.0..1000.0),
- generate_primitive::<Float64Type>(len, 0.5, -1000.0..1000.0),
+ generate_string_dictionary(rng, "prefix", 10, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 10, len, 0.5),
+ generate_string_dictionary(rng, "prefix", 100, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 100, len, 0.5),
+ generate_string_dictionary(rng, "prefix", 1000, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 1000, len, 0.5),
+ generate_strings(rng, 0..100, len, 1.0),
+ generate_strings(rng, 0..100, len, 0.5),
+ generate_primitive::<Int64Type>(rng, len, 1.0, -2000..2000),
+ generate_primitive::<Int64Type>(rng, len, 0.5, -2000..2000),
+ generate_primitive::<Float64Type>(rng, len, 1.0, -1000.0..1000.0),
+ generate_primitive::<Float64Type>(rng, len, 0.5, -1000.0..1000.0),
],
)
.unwrap()
}
fn generate_string_dictionary(
+ rng: &mut StdRng,
prefix: &str,
cardinality: usize,
len: usize,
valid_percent: f64,
) -> ArrayRef {
- let mut rng = rng();
let strings: Vec<_> = (0..cardinality).map(|x| format!("{prefix}#{x}")).collect();
Arc::new(DictionaryArray::<Int32Type>::from_iter((0..len).map(
@@ -110,11 +109,11 @@ fn generate_string_dictionary(
}
fn generate_strings(
+ rng: &mut StdRng,
string_length_range: Range<usize>,
len: usize,
valid_percent: f64,
) -> ArrayRef {
- let mut rng = rng();
Arc::new(StringArray::from_iter((0..len).map(|_| {
rng.random_bool(valid_percent).then(|| {
let string_len = rng.random_range(string_length_range.clone());
@@ -126,6 +125,7 @@ fn generate_strings(
}
fn generate_primitive<T>(
+ rng: &mut StdRng,
len: usize,
valid_percent: f64,
range: Range<T::Native>,
@@ -134,7 +134,6 @@ where
T: ArrowPrimitiveType,
T::Native: SampleUniform,
{
- let mut rng = rng();
Arc::new(PrimitiveArray::<T>::from_iter((0..len).map(|_| {
rng.random_bool(valid_percent)
.then(|| rng.random_range(range.clone()))
@@ -160,8 +159,9 @@ fn generate_file() -> NamedTempFile {
let mut writer =
ArrowWriter::try_new(&mut named_file, schema, Some(properties)).unwrap();
+ let mut rng = StdRng::seed_from_u64(0);
for _ in 0..NUM_BATCHES {
- let batch = generate_batch();
+ let batch = generate_batch(&mut rng);
writer.write(&batch).unwrap();
}
diff --git a/datafusion/core/benches/parquet_struct_query.rs b/datafusion/core/benches/parquet_struct_query.rs
index e7e91f0dd0..b7132973c1 100644
--- a/datafusion/core/benches/parquet_struct_query.rs
+++ b/datafusion/core/benches/parquet_struct_query.rs
@@ -27,7 +27,6 @@ use parquet::arrow::ArrowWriter;
use parquet::file::properties::{WriterProperties, WriterVersion};
use rand::distr::Alphanumeric;
use rand::prelude::*;
-use rand::rng;
use std::hint::black_box;
use std::ops::Range;
use std::path::Path;
@@ -59,8 +58,7 @@ fn schema() -> SchemaRef {
]))
}
-fn generate_strings(len: usize) -> ArrayRef {
- let mut rng = rng();
+fn generate_strings(rng: &mut StdRng, len: usize) -> ArrayRef {
Arc::new(StringArray::from_iter((0..len).map(|_| {
let string_len = rng.random_range(STRING_LENGTH_RANGE.clone());
Some(
@@ -71,7 +69,7 @@ fn generate_strings(len: usize) -> ArrayRef {
})))
}
-fn generate_batch(batch_id: usize) -> RecordBatch {
+fn generate_batch(rng: &mut StdRng, batch_id: usize) -> RecordBatch {
let schema = schema();
let len = WRITE_RECORD_BATCH_SIZE;
@@ -84,7 +82,7 @@ fn generate_batch(batch_id: usize) -> RecordBatch {
let struct_id_array = Arc::new(Int32Array::from(id_values));
// Generate random strings for struct value field
- let value_array = generate_strings(len);
+ let value_array = generate_strings(rng, len);
// Construct StructArray
let struct_array = StructArray::from(vec![
@@ -120,8 +118,9 @@ fn generate_file() -> NamedTempFile {
let mut writer =
ArrowWriter::try_new(&mut named_file, schema, Some(properties)).unwrap();
+ let mut rng = StdRng::seed_from_u64(0);
for batch_id in 0..NUM_BATCHES {
- let batch = generate_batch(batch_id);
+ let batch = generate_batch(&mut rng, batch_id);
writer.write(&batch).unwrap();
}
diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs
index 67e7f314d2..9cc4289ca1 100644
--- a/datafusion/functions-nested/benches/map.rs
+++ b/datafusion/functions-nested/benches/map.rs
@@ -28,8 +28,7 @@ use datafusion_expr::planner::ExprPlanner;
use datafusion_expr::{ColumnarValue, Expr, ScalarFunctionArgs};
use datafusion_functions_nested::map::map_udf;
use datafusion_functions_nested::planner::NestedFunctionPlanner;
-use rand::Rng;
-use rand::prelude::ThreadRng;
+use rand::prelude::*;
use std::collections::HashSet;
use std::hash::Hash;
use std::hint::black_box;
@@ -38,10 +37,7 @@ use std::sync::Arc;
const MAP_ROWS: usize = 1000;
const MAP_KEYS_PER_ROW: usize = 1000;
-fn gen_unique_values<T>(
- rng: &mut ThreadRng,
- mut make_value: impl FnMut(i32) -> T,
-) -> Vec<T>
+fn gen_unique_values<T>(rng: &mut StdRng, mut make_value: impl FnMut(i32) -> T) -> Vec<T>
where
T: Eq + Hash,
{
@@ -64,15 +60,15 @@ fn gen_repeat_values<T: Clone>(values: &[T], repeats: usize) -> Vec<T> {
repeated
}
-fn gen_utf8_values(rng: &mut ThreadRng) -> Vec<String> {
+fn gen_utf8_values(rng: &mut StdRng) -> Vec<String> {
gen_unique_values(rng, |value| value.to_string())
}
-fn gen_binary_values(rng: &mut ThreadRng) -> Vec<Vec<u8>> {
+fn gen_binary_values(rng: &mut StdRng) -> Vec<Vec<u8>> {
gen_unique_values(rng, |value| value.to_le_bytes().to_vec())
}
-fn gen_primitive_values(rng: &mut ThreadRng) -> Vec<i32> {
+fn gen_primitive_values(rng: &mut StdRng) -> Vec<i32> {
gen_unique_values(rng, |value| value)
}
@@ -122,7 +118,7 @@ fn bench_map_case(c: &mut Criterion, name: &str, keys: ArrayRef, values: ArrayRe
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("make_map_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let keys = gen_utf8_values(&mut rng);
let values = gen_primitive_values(&mut rng);
let mut buffer = Vec::new();
@@ -143,7 +139,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
});
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = Arc::new(Int32Array::from(gen_repeat_values(
&gen_primitive_values(&mut rng),
MAP_ROWS,
diff --git a/datafusion/functions/benches/concat.rs b/datafusion/functions/benches/concat.rs
index 0fb910800e..6736625be0 100644
--- a/datafusion/functions/benches/concat.rs
+++ b/datafusion/functions/benches/concat.rs
@@ -23,8 +23,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string::concat;
-use rand::Rng;
use rand::distr::Alphanumeric;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
@@ -48,17 +48,20 @@ fn create_array_args_view(size: usize) -> Vec<ColumnarValue> {
]
}
-fn generate_random_string(str_len: usize) -> String {
- rand::rng()
- .sample_iter(&Alphanumeric)
+fn generate_random_string(rng: &mut StdRng, str_len: usize) -> String {
+ rng.sample_iter(&Alphanumeric)
.take(str_len)
.map(char::from)
.collect()
}
-fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
+fn create_scalar_args(
+ rng: &mut StdRng,
+ count: usize,
+ str_len: usize,
+) -> Vec<ColumnarValue> {
std::iter::repeat_with(|| {
- let s = generate_random_string(str_len);
+ let s = generate_random_string(rng, str_len);
ColumnarValue::Scalar(ScalarValue::Utf8(Some(s)))
})
.take(count)
@@ -67,6 +70,7 @@ fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
fn criterion_benchmark(c: &mut Criterion) {
// Benchmark for array concat
+ let mut rng = StdRng::seed_from_u64(0);
for size in [1024, 4096, 8192] {
let args = create_array_args(size, 32);
let arg_fields = args
@@ -138,7 +142,7 @@ fn criterion_benchmark(c: &mut Criterion) {
}
// Benchmark for scalar concat
- let scalar_args = create_scalar_args(10, 100);
+ let scalar_args = create_scalar_args(&mut rng, 10, 100);
let scalar_arg_fields = scalar_args
.iter()
.enumerate()
diff --git a/datafusion/functions/benches/concat_ws.rs b/datafusion/functions/benches/concat_ws.rs
index 97d6d96411..d437f38773 100644
--- a/datafusion/functions/benches/concat_ws.rs
+++ b/datafusion/functions/benches/concat_ws.rs
@@ -23,8 +23,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string::concat_ws;
-use rand::Rng;
use rand::distr::Alphanumeric;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
@@ -38,9 +38,8 @@ fn create_array_args(size: usize, str_len: usize) -> Vec<ColumnarValue> {
]
}
-fn generate_random_string(str_len: usize) -> String {
- rand::rng()
- .sample_iter(&Alphanumeric)
+fn generate_random_string(rng: &mut StdRng, str_len: usize) -> String {
+ rng.sample_iter(&Alphanumeric)
.take(str_len)
.map(char::from)
.collect()
@@ -53,8 +52,9 @@ fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
",".to_string(),
))));
+ let mut rng = StdRng::seed_from_u64(0);
for _ in 0..count {
- let s = generate_random_string(str_len);
+ let s = generate_random_string(&mut rng, str_len);
args.push(ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))));
}
args
diff --git a/datafusion/functions/benches/date_bin.rs b/datafusion/functions/benches/date_bin.rs
index 28dee96987..bae1438fa4 100644
--- a/datafusion/functions/benches/date_bin.rs
+++ b/datafusion/functions/benches/date_bin.rs
@@ -25,10 +25,9 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::date_bin;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn timestamps(rng: &mut ThreadRng) -> TimestampSecondArray {
+fn timestamps(rng: &mut StdRng) -> TimestampSecondArray {
let mut seconds = vec![];
for _ in 0..1000 {
seconds.push(rng.random_range(0..1_000_000));
@@ -39,7 +38,7 @@ fn timestamps(rng: &mut ThreadRng) -> TimestampSecondArray {
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("date_bin_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let timestamps_array = Arc::new(timestamps(&mut rng)) as ArrayRef;
let batch_len = timestamps_array.len();
let interval = ColumnarValue::Scalar(ScalarValue::new_interval_dt(0, 1_000_000));
diff --git a/datafusion/functions/benches/gcd.rs b/datafusion/functions/benches/gcd.rs
index 3c72a46e66..ca49415b0f 100644
--- a/datafusion/functions/benches/gcd.rs
+++ b/datafusion/functions/benches/gcd.rs
@@ -25,12 +25,12 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::math::gcd;
-use rand::Rng;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
fn generate_i64_array(n_rows: usize) -> ArrayRef {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = (0..n_rows)
.map(|_| rng.random_range(0..1000))
.collect::<Vec<_>>();
diff --git a/datafusion/functions/benches/lcm.rs b/datafusion/functions/benches/lcm.rs
index 247c0ec749..5a4e5d2bce 100644
--- a/datafusion/functions/benches/lcm.rs
+++ b/datafusion/functions/benches/lcm.rs
@@ -24,12 +24,12 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::math::lcm;
-use rand::Rng;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
fn generate_i64_array(n_rows: usize) -> ArrayRef {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = (0..n_rows)
.map(|_| rng.random_range(0..1000))
.collect::<Vec<_>>();
diff --git a/datafusion/functions/benches/make_date.rs b/datafusion/functions/benches/make_date.rs
index 1c7b61ec60..2e82a871eb 100644
--- a/datafusion/functions/benches/make_date.rs
+++ b/datafusion/functions/benches/make_date.rs
@@ -25,10 +25,9 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::make_date;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn years(rng: &mut ThreadRng) -> Int32Array {
+fn years(rng: &mut StdRng) -> Int32Array {
let mut years = vec![];
for _ in 0..8192 {
years.push(rng.random_range(1900..2050));
@@ -37,7 +36,7 @@ fn years(rng: &mut ThreadRng) -> Int32Array {
Int32Array::from(years)
}
-fn months(rng: &mut ThreadRng) -> Int32Array {
+fn months(rng: &mut StdRng) -> Int32Array {
let mut months = vec![];
for _ in 0..8192 {
months.push(rng.random_range(1..13));
@@ -46,7 +45,7 @@ fn months(rng: &mut ThreadRng) -> Int32Array {
Int32Array::from(months)
}
-fn days(rng: &mut ThreadRng) -> Int32Array {
+fn days(rng: &mut StdRng) -> Int32Array {
let mut days = vec![];
for _ in 0..8192 {
days.push(rng.random_range(1..29));
@@ -56,7 +55,7 @@ fn days(rng: &mut ThreadRng) -> Int32Array {
}
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("make_date_col_col_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let years_array = Arc::new(years(&mut rng)) as ArrayRef;
let batch_len = years_array.len();
let years = ColumnarValue::Array(years_array);
@@ -86,7 +85,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("make_date_scalar_col_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let months_arr = Arc::new(months(&mut rng)) as ArrayRef;
let batch_len = months_arr.len();
@@ -116,7 +115,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("make_date_scalar_scalar_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let month = ColumnarValue::Scalar(ScalarValue::Int32(Some(11)));
let day_arr = Arc::new(days(&mut rng));
diff --git a/datafusion/functions/benches/pad.rs b/datafusion/functions/benches/pad.rs
index c71d5a7161..78ebf12236 100644
--- a/datafusion/functions/benches/pad.rs
+++ b/datafusion/functions/benches/pad.rs
@@ -28,8 +28,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::unicode;
-use rand::Rng;
-use rand::distr::{Distribution, Uniform};
+use rand::distr::Uniform;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
use std::time::Duration;
@@ -51,7 +51,7 @@ fn create_unicode_string_array<O: OffsetSizeTrait>(
size: usize,
null_density: f32,
) -> arrow::array::GenericStringArray<O> {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let mut builder = GenericStringBuilder::<O>::new();
for i in 0..size {
if rng.random::<f32>() < null_density {
@@ -67,7 +67,7 @@ fn create_unicode_string_view_array(
size: usize,
null_density: f32,
) -> arrow::array::StringViewArray {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let mut builder = StringViewBuilder::with_capacity(size);
for i in 0..size {
if rng.random::<f32>() < null_density {
@@ -104,7 +104,7 @@ where
dist: Uniform::new_inclusive::<i64, i64>(0, len as i64),
};
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
(0..size)
.map(|_| {
if rng.random::<f32>() < null_density {
diff --git a/datafusion/functions/benches/regx.rs b/datafusion/functions/benches/regx.rs
index a46b548236..dd263e41f6 100644
--- a/datafusion/functions/benches/regx.rs
+++ b/datafusion/functions/benches/regx.rs
@@ -32,11 +32,9 @@ use datafusion_functions::regex::regexpinstr::regexp_instr_func;
use datafusion_functions::regex::regexplike::{RegexpLikeFunc, regexp_like};
use datafusion_functions::regex::regexpmatch::regexp_match;
use datafusion_functions::regex::regexpreplace::regexp_replace;
-use rand::Rng;
use rand::distr::Alphanumeric;
-use rand::prelude::IndexedRandom;
-use rand::rngs::ThreadRng;
-fn data(rng: &mut ThreadRng) -> StringArray {
+use rand::prelude::*;
+fn data(rng: &mut StdRng) -> StringArray {
let mut data: Vec<String> = vec![];
for _ in 0..1000 {
data.push(
@@ -50,7 +48,7 @@ fn data(rng: &mut ThreadRng) -> StringArray {
StringArray::from(data)
}
-fn regex(rng: &mut ThreadRng) -> StringArray {
+fn regex(rng: &mut StdRng) -> StringArray {
let samples = [
".*([A-Z]{1}).*".to_string(),
"^(A).*".to_string(),
@@ -66,7 +64,7 @@ fn regex(rng: &mut ThreadRng) -> StringArray {
StringArray::from(data)
}
-fn start(rng: &mut ThreadRng) -> Int64Array {
+fn start(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -75,7 +73,7 @@ fn start(rng: &mut ThreadRng) -> Int64Array {
Int64Array::from(data)
}
-fn n(rng: &mut ThreadRng) -> Int64Array {
+fn n(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -84,7 +82,7 @@ fn n(rng: &mut ThreadRng) -> Int64Array {
Int64Array::from(data)
}
-fn flags(rng: &mut ThreadRng) -> StringArray {
+fn flags(rng: &mut StdRng) -> StringArray {
let samples = [Some("i".to_string()), Some("im".to_string()), None];
let mut sb = StringBuilder::new();
for _ in 0..1000 {
@@ -99,7 +97,7 @@ fn flags(rng: &mut ThreadRng) -> StringArray {
sb.finish()
}
-fn subexp(rng: &mut ThreadRng) -> Int64Array {
+fn subexp(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -112,7 +110,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let regexp_like_func = RegexpLikeFunc::new();
let config_options = Arc::new(ConfigOptions::default());
c.bench_function("regexp_count_1000 string", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -132,7 +130,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_count_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap();
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -152,7 +150,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_instr_1000 string", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -176,7 +174,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_instr_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap();
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -198,7 +196,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_like_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -212,7 +210,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_like_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
@@ -252,7 +250,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_match_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -270,7 +268,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_match_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
@@ -288,7 +286,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_replace_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -310,7 +308,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_replace_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
diff --git a/datafusion/functions/benches/to_char.rs b/datafusion/functions/benches/to_char.rs
index 350a55a371..8a9497bb33 100644
--- a/datafusion/functions/benches/to_char.rs
+++ b/datafusion/functions/benches/to_char.rs
@@ -27,12 +27,10 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_char;
-use rand::Rng;
-use rand::prelude::IndexedRandom;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
fn pick_date_in_range(
- rng: &mut ThreadRng,
+ rng: &mut StdRng,
start_date: NaiveDate,
end_date: NaiveDate,
) -> NaiveDate {
@@ -41,7 +39,7 @@ fn pick_date_in_range(
start_date + TimeDelta::try_days(random_days).unwrap()
}
-fn generate_date32_array(rng: &mut ThreadRng) -> Date32Array {
+fn generate_date32_array(rng: &mut StdRng) -> Date32Array {
let mut data: Vec<i32> = vec![];
let unix_days_from_ce = NaiveDate::from_ymd_opt(1970, 1, 1)
.unwrap()
@@ -62,7 +60,7 @@ fn generate_date32_array(rng: &mut ThreadRng) -> Date32Array {
Date32Array::from(data)
}
-fn generate_date64_array(rng: &mut ThreadRng) -> Date64Array {
+fn generate_date64_array(rng: &mut StdRng) -> Date64Array {
let start_date = "1970-01-01"
.parse::<NaiveDate>()
.expect("Date should parse");
@@ -96,21 +94,21 @@ const DATETIME_PATTERNS: [&str; 8] = [
"%c",
];
-fn pick_date_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_pattern(rng: &mut StdRng) -> String {
(*DATE_PATTERNS
.choose(rng)
.expect("Empty list of date patterns"))
.to_string()
}
-fn pick_date_time_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_time_pattern(rng: &mut StdRng) -> String {
(*DATETIME_PATTERNS
.choose(rng)
.expect("Empty list of date time patterns"))
.to_string()
}
-fn pick_date_and_date_time_mixed_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_and_date_time_mixed_pattern(rng: &mut StdRng) -> String {
match rng.random_bool(0.5) {
true => pick_date_pattern(rng),
false => pick_date_time_pattern(rng),
@@ -118,8 +116,8 @@ fn pick_date_and_date_time_mixed_pattern(rng: &mut ThreadRng) -> String {
}
fn generate_pattern_array(
- rng: &mut ThreadRng,
- pick_fn: impl Fn(&mut ThreadRng) -> String,
+ rng: &mut StdRng,
+ pick_fn: impl Fn(&mut StdRng) -> String,
) -> StringArray {
let mut data = Vec::with_capacity(1000);
@@ -130,15 +128,15 @@ fn generate_pattern_array(
StringArray::from(data)
}
-fn generate_date_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_date_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_pattern)
}
-fn generate_datetime_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_datetime_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_time_pattern)
}
-fn generate_mixed_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_mixed_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_and_date_time_mixed_pattern)
}
@@ -146,7 +144,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let config_options = Arc::new(ConfigOptions::default());
c.bench_function("to_char_array_date_only_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -173,7 +171,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_array_datetime_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -200,7 +198,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_array_mixed_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -227,7 +225,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_scalar_date_only_pattern_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -253,7 +251,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_scalar_datetime_pattern_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -285,7 +283,7 @@ fn criterion_benchmark(c: &mut Criterion) {
// Covers full fallback (every row triggers the cast)
c.bench_function("to_char_array_date32_datetime_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -313,7 +311,7 @@ fn criterion_benchmark(c: &mut Criterion) {
// Covers partial fallback (roughly half the rows trigger it)
c.bench_function("to_char_array_date32_mixed_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
diff --git a/datafusion/functions/benches/to_local_time.rs b/datafusion/functions/benches/to_local_time.rs
index 42d1e27198..04440bf0ac 100644
--- a/datafusion/functions/benches/to_local_time.rs
+++ b/datafusion/functions/benches/to_local_time.rs
@@ -24,17 +24,16 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_local_time;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn timestamps(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+fn timestamps(rng: &mut StdRng) -> TimestampNanosecondArray {
let nanos: Vec<i64> = (0..100_000)
.map(|_| rng.random_range(0..1_000_000_000_000_000_000i64))
.collect();
TimestampNanosecondArray::from(nanos).with_timezone("America/New_York")
}
-fn timestamps_with_nulls(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+fn timestamps_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray {
let values: Vec<Option<i64>> = (0..100_000)
.map(|_| {
if rng.random_range(0..10u32) == 0 {
@@ -73,7 +72,7 @@ fn bench_to_local_time(c: &mut Criterion, name: &str, array: ArrayRef) {
}
fn criterion_benchmark(c: &mut Criterion) {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
bench_to_local_time(
c,
"to_local_time_no_nulls_100k",
diff --git a/datafusion/functions/benches/to_time.rs b/datafusion/functions/benches/to_time.rs
index 6b3aa19241..f4499e2a7d 100644
--- a/datafusion/functions/benches/to_time.rs
+++ b/datafusion/functions/benches/to_time.rs
@@ -24,10 +24,9 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_time;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn random_time_string(rng: &mut ThreadRng) -> String {
+fn random_time_string(rng: &mut StdRng) -> String {
format!(
"{:02}:{:02}:{:02}.{:06}",
rng.random_range(0..24u32),
@@ -37,12 +36,12 @@ fn random_time_string(rng: &mut ThreadRng) -> String {
)
}
-fn time_strings(rng: &mut ThreadRng) -> StringArray {
+fn time_strings(rng: &mut StdRng) -> StringArray {
let strings: Vec<String> = (0..100_000).map(|_| random_time_string(rng)).collect();
StringArray::from(strings)
}
-fn time_strings_with_nulls(rng: &mut ThreadRng) -> StringArray {
+fn time_strings_with_nulls(rng: &mut StdRng) -> StringArray {
let values: Vec<Option<String>> = (0..100_000)
.map(|_| {
if rng.random_range(0..10u32) == 0 {
@@ -81,7 +80,7 @@ fn bench_to_time(c: &mut Criterion, name: &str, array: ArrayRef) {
}
fn criterion_benchmark(c: &mut Criterion) {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
bench_to_time(c, "to_time_no_nulls_100k", Arc::new(time_strings(&mut rng)));
bench_to_time(
c,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org