You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/04/08 15:31:28 UTC
(datafusion-comet) branch main updated: feat: add standalone shuffle benchmark tool (#3752)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 6260665d9 feat: add standalone shuffle benchmark tool (#3752)
6260665d9 is described below
commit 6260665d9b6c2c2e787cce05ed065d85924cf835
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Apr 8 09:31:23 2026 -0600
feat: add standalone shuffle benchmark tool (#3752)
---
native/Cargo.lock | 113 +++++-
native/shuffle/Cargo.toml | 11 +
native/shuffle/README.md | 41 ++
native/shuffle/src/bin/shuffle_bench.rs | 675 ++++++++++++++++++++++++++++++++
4 files changed, 826 insertions(+), 14 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 404656405..0c0c9e97b 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -120,12 +120,56 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+[[package]]
+name = "anstream"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
[[package]]
name = "anstyle"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+[[package]]
+name = "anstyle-parse"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
+dependencies = [
+ "anstyle",
+ "once_cell_polyfill",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "anyhow"
version = "1.0.102"
@@ -159,9 +203,9 @@ dependencies = [
[[package]]
name = "arc-swap"
-version = "1.9.0"
+version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
+checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
dependencies = [
"rustversion",
]
@@ -1355,6 +1399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
dependencies = [
"clap_builder",
+ "clap_derive",
]
[[package]]
@@ -1363,8 +1408,22 @@ version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
+ "anstream",
"anstyle",
"clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
]
[[package]]
@@ -1382,6 +1441,12 @@ dependencies = [
"cc",
]
+[[package]]
+name = "colorchoice"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+
[[package]]
name = "combine"
version = "4.6.7"
@@ -1968,6 +2033,7 @@ dependencies = [
"arrow",
"async-trait",
"bytes",
+ "clap",
"crc32c",
"crc32fast",
"criterion",
@@ -1980,6 +2046,7 @@ dependencies = [
"jni 0.21.1",
"log",
"lz4_flex",
+ "parquet",
"simd-adler32",
"snap",
"tempfile",
@@ -2850,9 +2917,9 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "2.3.0"
+version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "find-msvc-tools"
@@ -3690,6 +3757,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+
[[package]]
name = "itertools"
version = "0.13.0"
@@ -3999,9 +4072,9 @@ dependencies = [
[[package]]
name = "liblzma-sys"
-version = "0.4.5"
+version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186"
+checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
dependencies = [
"cc",
"libc",
@@ -4418,6 +4491,12 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+
[[package]]
name = "oorandom"
version = "11.1.5"
@@ -5729,9 +5808,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.27"
+version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "seq-macro"
@@ -6087,9 +6166,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symbolic-common"
-version = "12.17.3"
+version = "12.17.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ca086c1eb5c7ee74b151ba83c6487d5d33f8c08ad991b86f3f58f6629e68d5"
+checksum = "803d14d7cb9c6fa5b95a6f3de8af95b356a528d391998fa45a07d320a5573e51"
dependencies = [
"debugid",
"memmap2",
@@ -6099,9 +6178,9 @@ dependencies = [
[[package]]
name = "symbolic-demangle"
-version = "12.17.3"
+version = "12.17.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baa911a28a62823aaf2cc2e074212492a3ee69d0d926cc8f5b12b4a108ff5c0c"
+checksum = "39505731ae891b2dde47b0e4ae2ec40a7ced3476ab1129f1bf829e3fba62bb83"
dependencies = [
"cpp_demangle",
"rustc-demangle",
@@ -6338,9 +6417,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.51.0"
+version = "1.51.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd"
+checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c"
dependencies = [
"bytes",
"libc",
@@ -6634,6 +6713,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
[[package]]
name = "uuid"
version = "1.23.0"
diff --git a/native/shuffle/Cargo.toml b/native/shuffle/Cargo.toml
index 5cd7cd43e..505879e31 100644
--- a/native/shuffle/Cargo.toml
+++ b/native/shuffle/Cargo.toml
@@ -32,6 +32,7 @@ publish = false
arrow = { workspace = true }
async-trait = { workspace = true }
bytes = { workspace = true }
+clap = { version = "4", features = ["derive"], optional = true }
crc32c = "0.6.8"
crc32fast = "1.3.2"
datafusion = { workspace = true }
@@ -43,6 +44,8 @@ itertools = "0.14.0"
jni = "0.21"
log = "0.4"
lz4_flex = { version = "0.13.0", default-features = false, features = ["frame"] }
+# parquet is only used by the shuffle_bench binary (shuffle-bench feature)
+parquet = { workspace = true, optional = true }
simd-adler32 = "0.3.9"
snap = "1.1"
tokio = { version = "1", features = ["rt-multi-thread"] }
@@ -54,10 +57,18 @@ datafusion = { workspace = true, features = ["parquet_encryption", "sql"] }
itertools = "0.14.0"
tempfile = "3.26.0"
+[features]
+shuffle-bench = ["clap", "parquet"]
+
[lib]
name = "datafusion_comet_shuffle"
path = "src/lib.rs"
+[[bin]]
+name = "shuffle_bench"
+path = "src/bin/shuffle_bench.rs"
+required-features = ["shuffle-bench"]
+
[[bench]]
name = "shuffle_writer"
harness = false
diff --git a/native/shuffle/README.md b/native/shuffle/README.md
index 8fba6b032..0f53604fa 100644
--- a/native/shuffle/README.md
+++ b/native/shuffle/README.md
@@ -23,3 +23,44 @@ This crate provides the shuffle writer and reader implementation for Apache Data
of the [Apache DataFusion Comet] subproject.
[Apache DataFusion Comet]: https://github.com/apache/datafusion-comet/
+
+## Shuffle Benchmark Tool
+
+A standalone benchmark binary (`shuffle_bench`) is included for profiling shuffle write
+performance outside of Spark. It streams input data directly from Parquet files.
+
+### Basic usage
+
+```sh
+cargo run --release --features shuffle-bench --bin shuffle_bench -- \
+ --input /data/tpch-sf100/lineitem/ \
+ --partitions 200 \
+ --codec lz4 \
+ --hash-columns 0,3
+```
+
+### Options
+
+| Option | Default | Description |
+| --------------------- | -------------------------- | ------------------------------------------------------ |
+| `--input` | _(required)_ | Path to a Parquet file or directory of Parquet files |
+| `--partitions` | `200` | Number of output shuffle partitions |
+| `--partitioning` | `hash` | Partitioning scheme: `hash`, `single`, `round-robin` |
+| `--hash-columns` | `0` | Comma-separated column indices to hash on (e.g. `0,3`) |
+| `--codec` | `lz4` | Compression codec: `none`, `lz4`, `zstd`, `snappy` |
+| `--zstd-level` | `1` | Zstd compression level (1–22) |
+| `--batch-size` | `8192` | Batch size for reading Parquet data |
+| `--memory-limit` | _(none)_ | Memory limit in bytes; triggers spilling when exceeded |
+| `--write-buffer-size` | `1048576` | Write buffer size in bytes |
+| `--limit` | `0` | Limit rows processed per iteration (0 = no limit) |
+| `--iterations` | `1` | Number of timed iterations |
+| `--warmup` | `0` | Number of warmup iterations before timing |
+| `--output-dir` | `/tmp/comet_shuffle_bench` | Directory for temporary shuffle output files |
+
+### Profiling with flamegraph
+
+```sh
+cargo flamegraph --release --features shuffle-bench --bin shuffle_bench -- \
+ --input /data/tpch-sf100/lineitem/ \
+ --partitions 200 --codec lz4
+```
diff --git a/native/shuffle/src/bin/shuffle_bench.rs b/native/shuffle/src/bin/shuffle_bench.rs
new file mode 100644
index 000000000..bb8c2a038
--- /dev/null
+++ b/native/shuffle/src/bin/shuffle_bench.rs
@@ -0,0 +1,675 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Standalone shuffle benchmark tool for profiling Comet shuffle write
+//! performance outside of Spark. Streams input directly from Parquet files.
+//!
+//! # Usage
+//!
+//! ```sh
+//! cargo run --release --bin shuffle_bench -- \
+//! --input /data/tpch-sf100/lineitem/ \
+//! --partitions 200 \
+//! --codec lz4 \
+//! --hash-columns 0,3
+//! ```
+//!
+//! Profile with flamegraph:
+//! ```sh
+//! cargo flamegraph --release --bin shuffle_bench -- \
+//! --input /data/tpch-sf100/lineitem/ \
+//! --partitions 200 --codec lz4
+//! ```
+
+use arrow::datatypes::{DataType, SchemaRef};
+use clap::Parser;
+use datafusion::execution::config::SessionConfig;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_plan::common::collect;
+use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use datafusion_comet_shuffle::{CometPartitioning, CompressionCodec, ShuffleWriterExec};
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::Instant;
+
+#[derive(Parser, Debug)]
+#[command(
+ name = "shuffle_bench",
+ about = "Standalone benchmark for Comet shuffle write performance"
+)]
+struct Args {
+ /// Path to input Parquet file or directory of Parquet files
+ #[arg(long)]
+ input: PathBuf,
+
+ /// Batch size for reading Parquet data
+ #[arg(long, default_value_t = 8192)]
+ batch_size: usize,
+
+ /// Number of output shuffle partitions
+ #[arg(long, default_value_t = 200)]
+ partitions: usize,
+
+ /// Partitioning scheme: hash, single, round-robin
+ #[arg(long, default_value = "hash")]
+ partitioning: String,
+
+ /// Column indices to hash on (comma-separated, e.g. "0,3")
+ #[arg(long, default_value = "0")]
+ hash_columns: String,
+
+ /// Compression codec: none, lz4, zstd, snappy
+ #[arg(long, default_value = "lz4")]
+ codec: String,
+
+ /// Zstd compression level (1-22)
+ #[arg(long, default_value_t = 1)]
+ zstd_level: i32,
+
+ /// Memory limit in bytes (triggers spilling when exceeded)
+ #[arg(long)]
+ memory_limit: Option<usize>,
+
+ /// Number of iterations to run
+ #[arg(long, default_value_t = 1)]
+ iterations: usize,
+
+ /// Number of warmup iterations before timing
+ #[arg(long, default_value_t = 0)]
+ warmup: usize,
+
+ /// Output directory for shuffle data/index files
+ #[arg(long, default_value = "/tmp/comet_shuffle_bench")]
+ output_dir: PathBuf,
+
+ /// Write buffer size in bytes
+ #[arg(long, default_value_t = 1048576)]
+ write_buffer_size: usize,
+
+ /// Limit rows processed per iteration (0 = no limit)
+ #[arg(long, default_value_t = 0)]
+ limit: usize,
+
+ /// Number of concurrent shuffle tasks to simulate executor parallelism.
+ /// Each task reads the same input and writes to its own output files.
+ #[arg(long, default_value_t = 1)]
+ concurrent_tasks: usize,
+}
+
+fn main() {
+ let args = Args::parse();
+
+ // Create output directory
+ fs::create_dir_all(&args.output_dir).expect("Failed to create output directory");
+ let data_file = args.output_dir.join("data.out");
+ let index_file = args.output_dir.join("index.out");
+
+ let (schema, total_rows) = read_parquet_metadata(&args.input, args.limit);
+
+ let codec = parse_codec(&args.codec, args.zstd_level);
+ let hash_col_indices = parse_hash_columns(&args.hash_columns);
+
+ println!("=== Shuffle Benchmark ===");
+ println!("Input: {}", args.input.display());
+ println!(
+ "Schema: {} columns ({})",
+ schema.fields().len(),
+ describe_schema(&schema)
+ );
+ println!("Total rows: {}", format_number(total_rows as usize));
+ println!("Batch size: {}", format_number(args.batch_size));
+ println!("Partitioning: {}", args.partitioning);
+ println!("Partitions: {}", args.partitions);
+ println!("Codec: {:?}", codec);
+ println!("Hash columns: {:?}", hash_col_indices);
+ if let Some(mem_limit) = args.memory_limit {
+ println!("Memory limit: {}", format_bytes(mem_limit));
+ }
+ if args.concurrent_tasks > 1 {
+ println!("Concurrent: {} tasks", args.concurrent_tasks);
+ }
+ println!(
+ "Iterations: {} (warmup: {})",
+ args.iterations, args.warmup
+ );
+ println!();
+
+ let total_iters = args.warmup + args.iterations;
+ let mut write_times = Vec::with_capacity(args.iterations);
+ let mut data_file_sizes = Vec::with_capacity(args.iterations);
+ let mut last_metrics: Option<MetricsSet> = None;
+ let mut last_input_metrics: Option<MetricsSet> = None;
+
+ for i in 0..total_iters {
+ let is_warmup = i < args.warmup;
+ let label = if is_warmup {
+ format!("warmup {}/{}", i + 1, args.warmup)
+ } else {
+ format!("iter {}/{}", i - args.warmup + 1, args.iterations)
+ };
+
+ let (write_elapsed, metrics, input_metrics) = if args.concurrent_tasks > 1 {
+ let elapsed = run_concurrent_shuffle_writes(
+ &args.input,
+ &schema,
+ &codec,
+ &hash_col_indices,
+ &args,
+ );
+ (elapsed, None, None)
+ } else {
+ run_shuffle_write(
+ &args.input,
+ &schema,
+ &codec,
+ &hash_col_indices,
+ &args,
+ data_file.to_str().unwrap(),
+ index_file.to_str().unwrap(),
+ )
+ };
+ let data_size = fs::metadata(&data_file).map(|m| m.len()).unwrap_or(0);
+
+ if !is_warmup {
+ write_times.push(write_elapsed);
+ data_file_sizes.push(data_size);
+ last_metrics = metrics;
+ last_input_metrics = input_metrics;
+ }
+
+ print!(" [{label}] write: {:.3}s", write_elapsed);
+ if args.concurrent_tasks <= 1 {
+ print!(" output: {}", format_bytes(data_size as usize));
+ }
+
+ println!();
+ }
+
+ if args.iterations > 0 {
+ println!();
+ println!("=== Results ===");
+
+ let avg_write = write_times.iter().sum::<f64>() / write_times.len() as f64;
+ let write_throughput_rows = (total_rows as f64 * args.concurrent_tasks as f64) / avg_write;
+
+ println!("Write:");
+ println!(" avg time: {:.3}s", avg_write);
+ if write_times.len() > 1 {
+ let min = write_times.iter().cloned().fold(f64::INFINITY, f64::min);
+ let max = write_times
+ .iter()
+ .cloned()
+ .fold(f64::NEG_INFINITY, f64::max);
+ println!(" min/max: {:.3}s / {:.3}s", min, max);
+ }
+ println!(
+ " throughput: {} rows/s (total across {} tasks)",
+ format_number(write_throughput_rows as usize),
+ args.concurrent_tasks
+ );
+ if args.concurrent_tasks <= 1 {
+ let avg_data_size = data_file_sizes.iter().sum::<u64>() / data_file_sizes.len() as u64;
+ println!(
+ " output size: {}",
+ format_bytes(avg_data_size as usize)
+ );
+ }
+
+ if let Some(ref metrics) = last_input_metrics {
+ println!();
+ println!("Input Metrics (last iteration):");
+ print_input_metrics(metrics);
+ }
+
+ if let Some(ref metrics) = last_metrics {
+ println!();
+ println!("Shuffle Metrics (last iteration):");
+ print_shuffle_metrics(metrics, avg_write);
+ }
+ }
+
+ let _ = fs::remove_file(&data_file);
+ let _ = fs::remove_file(&index_file);
+}
+
+fn print_shuffle_metrics(metrics: &MetricsSet, total_wall_time_secs: f64) {
+ let get_metric = |name: &str| -> Option<usize> {
+ metrics
+ .iter()
+ .find(|m| m.value().name() == name)
+ .map(|m| m.value().as_usize())
+ };
+
+ let total_ns = (total_wall_time_secs * 1e9) as u64;
+ let fmt_time = |nanos: usize| -> String {
+ let secs = nanos as f64 / 1e9;
+ let pct = if total_ns > 0 {
+ (nanos as f64 / total_ns as f64) * 100.0
+ } else {
+ 0.0
+ };
+ format!("{:.3}s ({:.1}%)", secs, pct)
+ };
+
+ if let Some(input_batches) = get_metric("input_batches") {
+ println!(" input batches: {}", format_number(input_batches));
+ }
+ if let Some(nanos) = get_metric("repart_time") {
+ println!(" repart time: {}", fmt_time(nanos));
+ }
+ if let Some(nanos) = get_metric("encode_time") {
+ println!(" encode time: {}", fmt_time(nanos));
+ }
+ if let Some(nanos) = get_metric("write_time") {
+ println!(" write time: {}", fmt_time(nanos));
+ }
+
+ if let Some(spill_count) = get_metric("spill_count") {
+ if spill_count > 0 {
+ println!(" spill count: {}", format_number(spill_count));
+ }
+ }
+ if let Some(spilled_bytes) = get_metric("spilled_bytes") {
+ if spilled_bytes > 0 {
+ println!(" spilled bytes: {}", format_bytes(spilled_bytes));
+ }
+ }
+ if let Some(data_size) = get_metric("data_size") {
+ if data_size > 0 {
+ println!(" data size: {}", format_bytes(data_size));
+ }
+ }
+}
+
+fn print_input_metrics(metrics: &MetricsSet) {
+ let aggregated = metrics.aggregate_by_name();
+ for m in aggregated.iter() {
+ let value = m.value();
+ let name = value.name();
+ let v = value.as_usize();
+ if v == 0 {
+ continue;
+ }
+ // Format time metrics as seconds, everything else as a number
+ // Skip start/end timestamps — not useful in benchmark output
+ if matches!(
+ value,
+ MetricValue::StartTimestamp(_) | MetricValue::EndTimestamp(_)
+ ) {
+ continue;
+ }
+ let is_time = matches!(
+ value,
+ MetricValue::ElapsedCompute(_) | MetricValue::Time { .. }
+ );
+ if is_time {
+ println!(" {name}: {:.3}s", v as f64 / 1e9);
+ } else if name.contains("bytes") || name.contains("size") {
+ println!(" {name}: {}", format_bytes(v));
+ } else {
+ println!(" {name}: {}", format_number(v));
+ }
+ }
+}
+
+/// Read schema and total row count from Parquet metadata without loading any data.
+fn read_parquet_metadata(path: &Path, limit: usize) -> (SchemaRef, u64) {
+ let paths = collect_parquet_paths(path);
+ let mut schema = None;
+ let mut total_rows = 0u64;
+
+ for file_path in &paths {
+ let file = fs::File::open(file_path)
+ .unwrap_or_else(|e| panic!("Failed to open {}: {}", file_path.display(), e));
+ let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap_or_else(|e| {
+ panic!(
+ "Failed to read Parquet metadata from {}: {}",
+ file_path.display(),
+ e
+ )
+ });
+ if schema.is_none() {
+ schema = Some(Arc::clone(builder.schema()));
+ }
+ total_rows += builder.metadata().file_metadata().num_rows() as u64;
+ if limit > 0 && total_rows >= limit as u64 {
+ total_rows = total_rows.min(limit as u64);
+ break;
+ }
+ }
+
+ (schema.expect("No parquet files found"), total_rows)
+}
+
+fn collect_parquet_paths(path: &Path) -> Vec<PathBuf> {
+ if path.is_dir() {
+ let mut files: Vec<PathBuf> = fs::read_dir(path)
+ .unwrap_or_else(|e| panic!("Failed to read directory {}: {}", path.display(), e))
+ .filter_map(|entry| {
+ let p = entry.ok()?.path();
+ if p.extension().and_then(|e| e.to_str()) == Some("parquet") {
+ Some(p)
+ } else {
+ None
+ }
+ })
+ .collect();
+ files.sort();
+ if files.is_empty() {
+ panic!("No .parquet files found in {}", path.display());
+ }
+ files
+ } else {
+ vec![path.to_path_buf()]
+ }
+}
+
+fn run_shuffle_write(
+ input_path: &Path,
+ schema: &SchemaRef,
+ codec: &CompressionCodec,
+ hash_col_indices: &[usize],
+ args: &Args,
+ data_file: &str,
+ index_file: &str,
+) -> (f64, Option<MetricsSet>, Option<MetricsSet>) {
+ let partitioning = build_partitioning(
+ &args.partitioning,
+ args.partitions,
+ hash_col_indices,
+ schema,
+ );
+
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ rt.block_on(async {
+ let start = Instant::now();
+ let (shuffle_metrics, input_metrics) = execute_shuffle_write(
+ input_path.to_str().unwrap(),
+ codec.clone(),
+ partitioning,
+ args.batch_size,
+ args.memory_limit,
+ args.write_buffer_size,
+ args.limit,
+ data_file.to_string(),
+ index_file.to_string(),
+ )
+ .await
+ .unwrap();
+ (
+ start.elapsed().as_secs_f64(),
+ Some(shuffle_metrics),
+ Some(input_metrics),
+ )
+ })
+}
+
+/// Core async shuffle write logic shared by single and concurrent paths.
+#[allow(clippy::too_many_arguments)]
+async fn execute_shuffle_write(
+ input_path: &str,
+ codec: CompressionCodec,
+ partitioning: CometPartitioning,
+ batch_size: usize,
+ memory_limit: Option<usize>,
+ write_buffer_size: usize,
+ limit: usize,
+ data_file: String,
+ index_file: String,
+) -> datafusion::common::Result<(MetricsSet, MetricsSet)> {
+ let config = SessionConfig::new().with_batch_size(batch_size);
+ let mut runtime_builder = RuntimeEnvBuilder::new();
+ if let Some(mem_limit) = memory_limit {
+ runtime_builder = runtime_builder.with_memory_limit(mem_limit, 1.0);
+ }
+ let runtime_env = Arc::new(runtime_builder.build().unwrap());
+ let ctx = SessionContext::new_with_config_rt(config, runtime_env);
+
+ let mut df = ctx
+ .read_parquet(input_path, ParquetReadOptions::default())
+ .await
+ .expect("Failed to create Parquet scan");
+ if limit > 0 {
+ df = df.limit(0, Some(limit)).unwrap();
+ }
+
+ let parquet_plan = df
+ .create_physical_plan()
+ .await
+ .expect("Failed to create physical plan");
+
+ let input: Arc<dyn ExecutionPlan> = if parquet_plan
+ .properties()
+ .output_partitioning()
+ .partition_count()
+ > 1
+ {
+ Arc::new(CoalescePartitionsExec::new(parquet_plan))
+ } else {
+ parquet_plan
+ };
+
+ let exec = ShuffleWriterExec::try_new(
+ input,
+ partitioning,
+ codec,
+ data_file,
+ index_file,
+ false,
+ write_buffer_size,
+ )
+ .expect("Failed to create ShuffleWriterExec");
+
+ let task_ctx = ctx.task_ctx();
+ let stream = exec.execute(0, task_ctx).unwrap();
+ collect(stream).await.unwrap();
+
+ // Collect metrics from the input plan (Parquet scan + optional coalesce)
+ let input_metrics = collect_input_metrics(&exec);
+
+ Ok((exec.metrics().unwrap_or_default(), input_metrics))
+}
+
+/// Walk the plan tree and aggregate all metrics from input plans (everything below shuffle writer).
+fn collect_input_metrics(exec: &ShuffleWriterExec) -> MetricsSet {
+ let mut all_metrics = MetricsSet::new();
+ fn gather(plan: &dyn ExecutionPlan, out: &mut MetricsSet) {
+ if let Some(metrics) = plan.metrics() {
+ for m in metrics.iter() {
+ out.push(Arc::clone(m));
+ }
+ }
+ for child in plan.children() {
+ gather(child.as_ref(), out);
+ }
+ }
+ for child in exec.children() {
+ gather(child.as_ref(), &mut all_metrics);
+ }
+ all_metrics
+}
+
+/// Run N concurrent shuffle writes to simulate executor parallelism.
+/// Returns wall-clock time for all tasks to complete.
+fn run_concurrent_shuffle_writes(
+ input_path: &Path,
+ schema: &SchemaRef,
+ codec: &CompressionCodec,
+ hash_col_indices: &[usize],
+ args: &Args,
+) -> f64 {
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ rt.block_on(async {
+ let start = Instant::now();
+
+ let mut handles = Vec::with_capacity(args.concurrent_tasks);
+ for task_id in 0..args.concurrent_tasks {
+ let task_dir = args.output_dir.join(format!("task_{task_id}"));
+ fs::create_dir_all(&task_dir).expect("Failed to create task output directory");
+ let data_file = task_dir.join("data.out").to_str().unwrap().to_string();
+ let index_file = task_dir.join("index.out").to_str().unwrap().to_string();
+
+ let input_str = input_path.to_str().unwrap().to_string();
+ let codec = codec.clone();
+ let partitioning = build_partitioning(
+ &args.partitioning,
+ args.partitions,
+ hash_col_indices,
+ schema,
+ );
+ let batch_size = args.batch_size;
+ let memory_limit = args.memory_limit;
+ let write_buffer_size = args.write_buffer_size;
+ let limit = args.limit;
+
+ handles.push(tokio::spawn(async move {
+ execute_shuffle_write(
+ &input_str,
+ codec,
+ partitioning,
+ batch_size,
+ memory_limit,
+ write_buffer_size,
+ limit,
+ data_file,
+ index_file,
+ )
+ .await
+ .unwrap()
+ }));
+ }
+
+ for handle in handles {
+ handle.await.expect("Task panicked");
+ }
+
+ for task_id in 0..args.concurrent_tasks {
+ let task_dir = args.output_dir.join(format!("task_{task_id}"));
+ let _ = fs::remove_dir_all(&task_dir);
+ }
+
+ start.elapsed().as_secs_f64()
+ })
+}
+
+fn build_partitioning(
+ scheme: &str,
+ num_partitions: usize,
+ hash_col_indices: &[usize],
+ schema: &SchemaRef,
+) -> CometPartitioning {
+ match scheme {
+ "single" => CometPartitioning::SinglePartition,
+ "round-robin" => CometPartitioning::RoundRobin(num_partitions, 0),
+ "hash" => {
+ let exprs: Vec<Arc<dyn datafusion::physical_expr::PhysicalExpr>> = hash_col_indices
+ .iter()
+ .map(|&idx| {
+ let field = schema.field(idx);
+ Arc::new(Column::new(field.name(), idx))
+ as Arc<dyn datafusion::physical_expr::PhysicalExpr>
+ })
+ .collect();
+ CometPartitioning::Hash(exprs, num_partitions)
+ }
+ other => {
+ eprintln!("Unknown partitioning scheme: {other}. Using hash.");
+ build_partitioning("hash", num_partitions, hash_col_indices, schema)
+ }
+ }
+}
+
+fn parse_codec(codec: &str, zstd_level: i32) -> CompressionCodec {
+ match codec.to_lowercase().as_str() {
+ "none" => CompressionCodec::None,
+ "lz4" => CompressionCodec::Lz4Frame,
+ "zstd" => CompressionCodec::Zstd(zstd_level),
+ "snappy" => CompressionCodec::Snappy,
+ other => {
+ eprintln!("Unknown codec: {other}. Using zstd.");
+ CompressionCodec::Zstd(zstd_level)
+ }
+ }
+}
+
+fn parse_hash_columns(s: &str) -> Vec<usize> {
+ s.split(',')
+ .filter(|s| !s.is_empty())
+ .map(|s| s.trim().parse::<usize>().expect("Invalid column index"))
+ .collect()
+}
+
+fn describe_schema(schema: &arrow::datatypes::Schema) -> String {
+ let mut counts = std::collections::HashMap::new();
+ for field in schema.fields() {
+ let type_name = match field.data_type() {
+ DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64
+ | DataType::UInt8
+ | DataType::UInt16
+ | DataType::UInt32
+ | DataType::UInt64 => "int",
+ DataType::Float16 | DataType::Float32 | DataType::Float64 => "float",
+ DataType::Utf8 | DataType::LargeUtf8 => "string",
+ DataType::Boolean => "bool",
+ DataType::Date32 | DataType::Date64 => "date",
+ DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => "decimal",
+ DataType::Timestamp(_, _) => "timestamp",
+ DataType::Binary | DataType::LargeBinary | DataType::FixedSizeBinary(_) => "binary",
+ _ => "other",
+ };
+ *counts.entry(type_name).or_insert(0) += 1;
+ }
+ let mut parts: Vec<String> = counts
+ .into_iter()
+ .map(|(k, v)| format!("{}x{}", v, k))
+ .collect();
+ parts.sort();
+ parts.join(", ")
+}
+
+fn format_number(n: usize) -> String {
+ let s = n.to_string();
+ let mut result = String::new();
+ for (i, c) in s.chars().rev().enumerate() {
+ if i > 0 && i % 3 == 0 {
+ result.push(',');
+ }
+ result.push(c);
+ }
+ result.chars().rev().collect()
+}
+
+fn format_bytes(bytes: usize) -> String {
+ if bytes >= 1024 * 1024 * 1024 {
+ format!("{:.2} GiB", bytes as f64 / (1024.0 * 1024.0 * 1024.0))
+ } else if bytes >= 1024 * 1024 {
+ format!("{:.2} MiB", bytes as f64 / (1024.0 * 1024.0))
+ } else if bytes >= 1024 {
+ format!("{:.2} KiB", bytes as f64 / 1024.0)
+ } else {
+ format!("{bytes} B")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org