You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/02 21:42:10 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6541-97ea38f838aa705e3c9465e24734d6df0ffc976b created (now 90e4276ff0)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6541-97ea38f838aa705e3c9465e24734d6df0ffc976b
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 90e4276ff0 add benches remainig scalars (#6541)
This branch includes the following new commits:
new 90e4276ff0 add benches remainig scalars (#6541)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: add benches remainig scalars (#6541)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6541-97ea38f838aa705e3c9465e24734d6df0ffc976b
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 90e4276ff03c6743c0e12bcc75a55a333c2528f7
Author: Bhargava Vadlamani <11...@users.noreply.github.com>
AuthorDate: Fri Oct 2 21:41:48 2026 +0000
add benches remainig scalars (#6541)
---
native/spark-expr/Cargo.toml | 16 +++++
native/spark-expr/benches/concat_ws.rs | 65 +++++++++++++++++++
native/spark-expr/benches/list_positions.rs | 55 ++++++++++++++++
native/spark-expr/benches/map_from_arrays.rs | 75 ++++++++++++++++++++++
native/spark-expr/benches/sqrt.rs | 44 +++++++++++++
native/spark-expr/src/lib.rs | 2 +-
native/spark-expr/src/map_funcs/map_from_arrays.rs | 2 +-
native/spark-expr/src/map_funcs/mod.rs | 2 +-
8 files changed, 258 insertions(+), 3 deletions(-)
diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml
index a97334b56f..0857fa2399 100644
--- a/native/spark-expr/Cargo.toml
+++ b/native/spark-expr/Cargo.toml
@@ -346,6 +346,22 @@ harness = false
name = "abs"
harness = false
+[[bench]]
+name = "sqrt"
+harness = false
+
+[[bench]]
+name = "concat_ws"
+harness = false
+
+[[bench]]
+name = "list_positions"
+harness = false
+
+[[bench]]
+name = "map_from_arrays"
+harness = false
+
[[bench]]
name = "murmur3"
harness = false
diff --git a/native/spark-expr/benches/concat_ws.rs b/native/spark-expr/benches/concat_ws.rs
new file mode 100644
index 0000000000..710a482ed9
--- /dev/null
+++ b/native/spark-expr/benches/concat_ws.rs
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::datatypes::{DataType, Field};
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::ScalarValue;
+use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_comet_spark_expr::CometConcatWs;
+use std::hint::black_box;
+use std::sync::Arc;
+
+#[path = "common/mod.rs"]
+mod common;
+use common::{string_array, NULL_RATIOS, ROW_COUNTS};
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let udf = CometConcatWs::default();
+ let mut group = c.benchmark_group("concat_ws");
+ for rows in ROW_COUNTS {
+ for (null_ratio, tag) in NULL_RATIOS {
+ let args = vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("-".to_string()))),
+ ColumnarValue::Array(string_array(rows, null_ratio, |i| format!("a{}", i % 1000))),
+ ColumnarValue::Array(string_array(rows, null_ratio, |i| format!("b{}", i % 1000))),
+ ];
+ group.bench_with_input(
+ BenchmarkId::from_parameter(format!("{rows}/{tag}")),
+ &args,
+ |b, args| {
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.to_vec(),
+ arg_fields: vec![],
+ number_rows: rows,
+ return_field: Arc::new(Field::new("result", DataType::Utf8, true)),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/native/spark-expr/benches/list_positions.rs b/native/spark-expr/benches/list_positions.rs
new file mode 100644
index 0000000000..e49685fa76
--- /dev/null
+++ b/native/spark-expr/benches/list_positions.rs
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, RecordBatch};
+use arrow::datatypes::{Field, Schema};
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion_comet_spark_expr::ListPositionsExpr;
+use std::hint::black_box;
+use std::sync::Arc;
+
+#[path = "common/mod.rs"]
+mod common;
+use common::{list_arrays, NULL_RATIOS, ROW_COUNTS};
+
+fn single_col_batch(col: ArrayRef) -> RecordBatch {
+ let schema = Schema::new(vec![Field::new("c0", col.data_type().clone(), true)]);
+ RecordBatch::try_new(Arc::new(schema), vec![col]).unwrap()
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let expr = ListPositionsExpr::new(Arc::new(Column::new("c0", 0)));
+ let mut group = c.benchmark_group("list_positions");
+ for rows in ROW_COUNTS {
+ for (null_ratio, tag) in NULL_RATIOS {
+ for (ty, list, _) in list_arrays(rows, null_ratio, 8) {
+ let batch = single_col_batch(list);
+ group.bench_with_input(
+ BenchmarkId::from_parameter(format!("{ty}/{rows}/{tag}")),
+ &batch,
+ |b, batch| b.iter(|| black_box(expr.evaluate(batch).unwrap())),
+ );
+ }
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/native/spark-expr/benches/map_from_arrays.rs b/native/spark-expr/benches/map_from_arrays.rs
new file mode 100644
index 0000000000..e7a146f626
--- /dev/null
+++ b/native/spark-expr/benches/map_from_arrays.rs
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::builder::{ListBuilder, StringBuilder};
+use arrow::array::ArrayRef;
+use arrow::datatypes::Field;
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use datafusion::common::config::ConfigOptions;
+use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_comet_spark_expr::SparkMapFromArrays;
+use std::hint::black_box;
+use std::sync::Arc;
+
+const BATCH_SIZE: usize = 8192;
+
+/// A `List<Utf8>` column of `BATCH_SIZE` rows, each holding `entries` strings. Values are made
+/// distinct per row so the map constructor never hits a duplicate key.
+fn string_list(entries: usize, prefix: &str) -> ArrayRef {
+ let mut builder = ListBuilder::new(StringBuilder::new());
+ for row in 0..BATCH_SIZE {
+ for e in 0..entries {
+ builder.values().append_value(format!("{prefix}{row}_{e}"));
+ }
+ builder.append(true);
+ }
+ Arc::new(builder.finish())
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let udf = SparkMapFromArrays::default();
+ let mut group = c.benchmark_group("map_from_arrays");
+ for entries in [2usize, 8, 32] {
+ let keys = string_list(entries, "k");
+ let values = string_list(entries, "v");
+ let return_type = udf
+ .return_type(&[keys.data_type().clone(), values.data_type().clone()])
+ .unwrap();
+ let args = vec![
+ ColumnarValue::Array(Arc::clone(&keys)),
+ ColumnarValue::Array(Arc::clone(&values)),
+ ];
+ group.bench_with_input(BenchmarkId::from_parameter(entries), &args, |b, args| {
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.to_vec(),
+ arg_fields: vec![],
+ number_rows: BATCH_SIZE,
+ return_field: Arc::new(Field::new("result", return_type.clone(), true)),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ });
+ }
+ group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/native/spark-expr/benches/sqrt.rs b/native/spark-expr/benches/sqrt.rs
new file mode 100644
index 0000000000..cb7757ce34
--- /dev/null
+++ b/native/spark-expr/benches/sqrt.rs
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use datafusion::physical_plan::ColumnarValue;
+use datafusion_comet_spark_expr::spark_sqrt;
+use std::hint::black_box;
+
+#[path = "common/mod.rs"]
+mod common;
+use common::{f64_array, NULL_RATIOS, ROW_COUNTS};
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let mut group = c.benchmark_group("sqrt");
+ for rows in ROW_COUNTS {
+ for (null_ratio, tag) in NULL_RATIOS {
+ let arr = f64_array(rows, null_ratio, |i| (i % 1000) as f64);
+ let args = vec![ColumnarValue::Array(arr)];
+ group.bench_with_input(
+ BenchmarkId::from_parameter(format!("{rows}/{tag}")),
+ &args,
+ |b, args| b.iter(|| black_box(spark_sqrt(black_box(args)).unwrap())),
+ );
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs
index dc1b8272d6..991fd5e52e 100644
--- a/native/spark-expr/src/lib.rs
+++ b/native/spark-expr/src/lib.rs
@@ -70,7 +70,7 @@ mod hll_scalar;
pub use hll_scalar::spark_hll_sketch_estimate;
pub use hll_scalar::spark_hll_union;
mod map_funcs;
-pub use map_funcs::{spark_map_sort, SparkMapExtract};
+pub use map_funcs::{spark_map_sort, SparkMapExtract, SparkMapFromArrays};
mod math_funcs;
mod nondetermenistic_funcs;
pub mod url_funcs;
diff --git a/native/spark-expr/src/map_funcs/map_from_arrays.rs b/native/spark-expr/src/map_funcs/map_from_arrays.rs
index c5a9d523bb..14d2da32a4 100644
--- a/native/spark-expr/src/map_funcs/map_from_arrays.rs
+++ b/native/spark-expr/src/map_funcs/map_from_arrays.rs
@@ -25,7 +25,7 @@ use crate::SparkError;
/// Checks row boundaries before the DataFusion `map` used by CometMapFromArrays.
#[derive(Debug, Default, PartialEq, Eq, Hash)]
-pub(crate) struct SparkMapFromArrays {
+pub struct SparkMapFromArrays {
inner: MapFunc,
}
diff --git a/native/spark-expr/src/map_funcs/mod.rs b/native/spark-expr/src/map_funcs/mod.rs
index d1a84009b5..ddea77c932 100644
--- a/native/spark-expr/src/map_funcs/mod.rs
+++ b/native/spark-expr/src/map_funcs/mod.rs
@@ -19,5 +19,5 @@ mod map_extract;
mod map_from_arrays;
mod map_sort;
pub use map_extract::SparkMapExtract;
-pub(crate) use map_from_arrays::SparkMapFromArrays;
+pub use map_from_arrays::SparkMapFromArrays;
pub use map_sort::spark_map_sort;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org