You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/05/01 17:13:42 UTC
(datafusion-ballista) branch main updated: chore: remove NYC Taxi benchmark (#1644)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 9f4055646 chore: remove NYC Taxi benchmark (#1644)
9f4055646 is described below
commit 9f4055646d3329e2003d1845c90a1385fa9b9dcf
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri May 1 11:13:38 2026 -0600
chore: remove NYC Taxi benchmark (#1644)
---
benchmarks/README.md | 19 ----
benchmarks/src/bin/nyctaxi.rs | 157 ----------------------------
docs/source/user-guide/python/jupyter.md | 2 +-
docs/source/user-guide/python/quickstart.md | 4 +-
4 files changed, 3 insertions(+), 179 deletions(-)
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 93f899ec9..caf503296 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -258,24 +258,6 @@ $SPARK_HOME/bin/spark-submit \
--query 1
```
-## NYC Taxi Benchmark
-
-These benchmarks are based on the [New York Taxi and Limousine Commission][2] data set.
-
-```bash
-cargo run --release --bin nyctaxi -- --iterations 3 --path /mnt/nyctaxi/csv --format csv --batch-size 4096
-```
-
-Example output:
-
-```bash
-Running benchmarks with the following options: Opt { debug: false, iterations: 3, batch_size: 4096, path: "/mnt/nyctaxi/csv", file_format: "csv" }
-Executing 'fare_amt_by_passenger'
-Query 'fare_amt_by_passenger' iteration 0 took 7138 ms
-Query 'fare_amt_by_passenger' iteration 1 took 7599 ms
-Query 'fare_amt_by_passenger' iteration 2 took 7969 ms
-```
-
## Running the Ballista Loadtest
```bash
@@ -292,4 +274,3 @@ Query 'fare_amt_by_passenger' iteration 2 took 7969 ms
```
[1]: http://www.tpc.org/tpch/
-[2]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
diff --git a/benchmarks/src/bin/nyctaxi.rs b/benchmarks/src/bin/nyctaxi.rs
deleted file mode 100644
index 716707341..000000000
--- a/benchmarks/src/bin/nyctaxi.rs
+++ /dev/null
@@ -1,157 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Apache Arrow Rust Benchmarks
-
-use std::collections::HashMap;
-use std::path::PathBuf;
-use std::process;
-use std::time::Instant;
-
-use datafusion::arrow::datatypes::{DataType, Field, Schema};
-use datafusion::arrow::util::pretty;
-
-use datafusion::error::Result;
-use datafusion::execution::context::{SessionConfig, SessionContext};
-
-use datafusion::physical_plan::collect;
-use datafusion::prelude::{CsvReadOptions, ParquetReadOptions};
-use structopt::StructOpt;
-
-#[cfg(feature = "mimalloc")]
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-#[derive(Debug, StructOpt)]
-#[structopt(name = "Benchmarks", about = "Apache Arrow Rust Benchmarks.")]
-struct Opt {
- /// Activate debug mode to see query results
- #[structopt(short, long)]
- debug: bool,
-
- /// Number of iterations of each test run
- #[structopt(short = "i", long = "iterations", default_value = "3")]
- iterations: usize,
-
- /// Number of partitions to process in parallel
- #[structopt(short = "p", long = "partitions", default_value = "2")]
- partitions: usize,
-
- /// Batch size when reading CSV or Parquet files
- #[structopt(short = "s", long = "batch-size", default_value = "8192")]
- batch_size: usize,
-
- /// Path to data files
- #[structopt(parse(from_os_str), required = true, short = "p", long = "path")]
- path: PathBuf,
-
- /// File format: `csv` or `parquet`
- #[structopt(short = "f", long = "format", default_value = "csv")]
- file_format: String,
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
- let opt = Opt::from_args();
- println!("Running benchmarks with the following options: {opt:?}");
-
- let config = SessionConfig::new()
- .with_target_partitions(opt.partitions)
- .with_batch_size(opt.batch_size);
- let mut ctx = SessionContext::new_with_config(config);
-
- let path = opt.path.to_str().unwrap();
-
- match opt.file_format.as_str() {
- "csv" => {
- let schema = nyctaxi_schema();
- let options = CsvReadOptions::new().schema(&schema).has_header(true);
- ctx.register_csv("tripdata", path, options).await?
- }
- "parquet" => {
- ctx.register_parquet("tripdata", path, ParquetReadOptions::default())
- .await?
- }
- other => {
- println!("Invalid file format '{other}'");
- process::exit(-1);
- }
- }
-
- datafusion_sql_benchmarks(&mut ctx, opt.iterations, opt.debug).await
-}
-
-async fn datafusion_sql_benchmarks(
- ctx: &mut SessionContext,
- iterations: usize,
- debug: bool,
-) -> Result<()> {
- let mut queries = HashMap::new();
- queries.insert("fare_amt_by_passenger", "SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM tripdata GROUP BY passenger_count");
- for (name, sql) in &queries {
- println!("Executing '{name}'");
- for i in 0..iterations {
- let start = Instant::now();
- execute_sql(ctx, sql, debug).await?;
- println!(
- "Query '{}' iteration {} took {} ms",
- name,
- i,
- start.elapsed().as_millis()
- );
- }
- }
- Ok(())
-}
-
-async fn execute_sql(ctx: &SessionContext, sql: &str, debug: bool) -> Result<()> {
- let session_state = ctx.state();
- let plan = session_state.create_logical_plan(sql).await?;
- let plan = session_state.optimize(&plan)?;
- if debug {
- println!("Optimized logical plan:\n{plan:?}");
- }
- let physical_plan = session_state.create_physical_plan(&plan).await?;
- let task_ctx = ctx.task_ctx();
- let result = collect(physical_plan, task_ctx).await?;
- if debug {
- pretty::print_batches(&result)?;
- }
- Ok(())
-}
-
-fn nyctaxi_schema() -> Schema {
- Schema::new(vec![
- Field::new("VendorID", DataType::Utf8, true),
- Field::new("tpep_pickup_datetime", DataType::Utf8, true),
- Field::new("tpep_dropoff_datetime", DataType::Utf8, true),
- Field::new("passenger_count", DataType::Int32, true),
- Field::new("trip_distance", DataType::Utf8, true),
- Field::new("RatecodeID", DataType::Utf8, true),
- Field::new("store_and_fwd_flag", DataType::Utf8, true),
- Field::new("PULocationID", DataType::Utf8, true),
- Field::new("DOLocationID", DataType::Utf8, true),
- Field::new("payment_type", DataType::Utf8, true),
- Field::new("fare_amount", DataType::Float64, true),
- Field::new("extra", DataType::Float64, true),
- Field::new("mta_tax", DataType::Float64, true),
- Field::new("tip_amount", DataType::Float64, true),
- Field::new("tolls_amount", DataType::Float64, true),
- Field::new("improvement_surcharge", DataType::Float64, true),
- Field::new("total_amount", DataType::Float64, true),
- ])
-}
diff --git a/docs/source/user-guide/python/jupyter.md b/docs/source/user-guide/python/jupyter.md
index d1703817b..1a978c2e7 100644
--- a/docs/source/user-guide/python/jupyter.md
+++ b/docs/source/user-guide/python/jupyter.md
@@ -29,7 +29,7 @@ from ballista import BallistaSessionContext
ctx = BallistaSessionContext("df://localhost:50050")
-ctx.register_parquet("trips", "/path/to/nyctaxi.parquet")
+ctx.register_parquet("trips", "/path/to/trips.parquet")
# The result renders as an HTML table when this is the last expression in a cell
ctx.sql("SELECT * FROM trips LIMIT 10")
diff --git a/docs/source/user-guide/python/quickstart.md b/docs/source/user-guide/python/quickstart.md
index 733ee7682..012e4006d 100644
--- a/docs/source/user-guide/python/quickstart.md
+++ b/docs/source/user-guide/python/quickstart.md
@@ -87,11 +87,11 @@ Before running SQL queries, register tables with the context using a `register_*
method or a `CREATE EXTERNAL TABLE` statement:
```python
-ctx.register_parquet("trips", "/mnt/bigdata/nyctaxi")
+ctx.register_parquet("trips", "/mnt/bigdata/trips")
```
```python
-ctx.sql("CREATE EXTERNAL TABLE trips STORED AS PARQUET LOCATION '/mnt/bigdata/nyctaxi'")
+ctx.sql("CREATE EXTERNAL TABLE trips STORED AS PARQUET LOCATION '/mnt/bigdata/trips'")
```
### Executing Queries
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org