You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/17 02:50:13 UTC
(datafusion) branch gh-readonly-queue/main/pr-24323-bbf739a3cfe1d5cc7646e37250c947c6a56b5b0e created (now 710e74e676)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-24323-bbf739a3cfe1d5cc7646e37250c947c6a56b5b0e
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 710e74e676 Add focused benchmark for Parquet pruning setup cache (#24323)
This branch includes the following new commits:
new 710e74e676 Add focused benchmark for Parquet pruning setup cache (#24323)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: Add focused benchmark for Parquet pruning setup cache (#24323)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24323-bbf739a3cfe1d5cc7646e37250c947c6a56b5b0e
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 710e74e6764275675b117823d26d4944115f6ba2
Author: kosiew <ko...@gmail.com>
AuthorDate: Mon Aug 17 02:49:50 2026 +0000
Add focused benchmark for Parquet pruning setup cache (#24323)
## Which issue does this PR close?
* Part of #21554
## Rationale for this change
`ParquetPruningSetupCache` in #21566 needs a focused benchmark that
exercises it under deliberately cache-favourable conditions.
This benchmark provides a workload where many Parquet files share the
same physical schema and use the same predicate, target partition count,
and query, creating repeated opportunities to reuse pruning setup. It is
intended to measure end-to-end scan cost for this specific workload
shape, rather than represent general DataFusion performance or predict
improvements in broader benchmarks such as ClickBench.
## What changes are included in this PR?
* Adds a Criterion benchmark that creates 128 Parquet files sharing one
physical schema and repeatedly executes `SELECT id FROM t WHERE id >=
0`.
* Keeps file generation, runtime creation, table registration, and
initial correctness validation outside the timed Criterion loop.
* Validates that the query returns all 16,384 expected rows before
benchmarking.
* Enables Criterion's `async_tokio` feature so the asynchronous query
execution can be benchmarked with the Tokio runtime.
* Registers the new `parquet_pruning_setup_cache` benchmark target.
* Documents that the benchmark is intentionally cache-favourable and
that its results should not be interpreted as general DataFusion or
ClickBench performance results.
## Are these changes tested?
The benchmark includes a correctness check that verifies the query
returns `128 × 128 = 16,384` rows before the timed benchmark begins.
No additional tests are included in this patch.
## Are there any user-facing changes?
No. This PR adds a focused benchmark and benchmark configuration only;
it does not change user-facing DataFusion behavior or public APIs.
## LLM-generated code disclosure
This PR includes LLM-generated code and comments. All LLM-generated
content has been manually reviewed.
---
benchmarks/Cargo.toml | 6 +-
benchmarks/benches/parquet_pruning_setup_cache.rs | 110 ++++++++++++++++++++++
2 files changed, 115 insertions(+), 1 deletion(-)
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index 62eea98439..cd222c3a5e 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -44,7 +44,7 @@ arrow = { workspace = true }
async-trait = "0.1"
bytes = { workspace = true }
clap = { version = "4.6.0", features = ["derive", "env", "string"] }
-criterion = { workspace = true, features = ["html_reports"] }
+criterion = { workspace = true, features = ["async_tokio", "html_reports"] }
datafusion = { workspace = true, default-features = true }
datafusion-common = { workspace = true, default-features = true }
datafusion-common-runtime = { workspace = true }
@@ -71,3 +71,7 @@ tempfile = { workspace = true }
[[bench]]
harness = false
name = "sql"
+
+[[bench]]
+harness = false
+name = "parquet_pruning_setup_cache"
diff --git a/benchmarks/benches/parquet_pruning_setup_cache.rs b/benchmarks/benches/parquet_pruning_setup_cache.rs
new file mode 100644
index 0000000000..685ad698fb
--- /dev/null
+++ b/benchmarks/benches/parquet_pruning_setup_cache.rs
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks end-to-end Parquet scan cost for a cache-favourable workload.
+//!
+//! The 128 files share one physical schema and use the same predicate and target
+//! partition count, so pruning setup is reusable in a cache-enabled comparison
+//! branch. The predicate matches every file, so each scan must adapt the
+//! predicate and build (or reuse) pruning setup for all files.
+//!
+//! This is a focused baseline for comparing cache-disabled and cache-enabled
+//! branches. It is not a general DataFusion benchmark and must not be
+//! interpreted as a ClickBench performance result.
+
+use std::{fs::File, sync::Arc};
+
+use arrow::{
+ array::Int64Array,
+ datatypes::{DataType, Field, Schema},
+ record_batch::RecordBatch,
+};
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion::prelude::{SessionConfig, SessionContext};
+use parquet::arrow::ArrowWriter;
+use tempfile::TempDir;
+
+const FILES: usize = 128;
+const ROWS_PER_FILE: usize = 128;
+
+fn write_files() -> TempDir {
+ let directory = tempfile::tempdir().unwrap();
+ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+
+ for file_index in 0..FILES {
+ let start = i64::try_from(file_index * ROWS_PER_FILE).unwrap();
+ let values = Int64Array::from_iter_values(
+ start..start + i64::try_from(ROWS_PER_FILE).unwrap(),
+ );
+ let batch =
+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)]).unwrap();
+ let file =
+ File::create(directory.path().join(format!("{file_index}.parquet"))).unwrap();
+ let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ }
+
+ directory
+}
+
+fn criterion_benchmark(criterion: &mut Criterion) {
+ let directory = write_files();
+ let runtime = tokio::runtime::Runtime::new().unwrap();
+ let context = runtime.block_on(async {
+ let context = SessionContext::new_with_config(
+ SessionConfig::new().with_target_partitions(1),
+ );
+ context
+ .register_parquet("t", directory.path().to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ context
+ });
+
+ runtime.block_on(async {
+ let batches = context
+ .sql("SELECT id FROM t WHERE id >= 0")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_eq!(
+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
+ FILES * ROWS_PER_FILE
+ );
+ });
+
+ criterion.bench_function(
+ "parquet_pruning_setup_cache/same_schema_files",
+ |bencher| {
+ bencher.to_async(&runtime).iter(|| async {
+ context
+ .sql("SELECT id FROM t WHERE id >= 0")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap()
+ });
+ },
+ );
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org