You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2026/03/23 03:02:19 UTC
(sedona-spatialbench) 01/01: Update DuckDB to 1.5.0, default to SF10 with 1800s timeout, and add crash resilience
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch update/duckdb-150-benchmark-defaults
in repository https://gitbox.apache.org/repos/asf/sedona-spatialbench.git
commit 1468bc48b7aa4a652df4783e5c5d1477867d3939
Author: Jia Yu <ji...@gmail.com>
AuthorDate: Sun Mar 22 20:02:04 2026 -0700
Update DuckDB to 1.5.0, default to SF10 with 1800s timeout, and add crash resilience
- Remove DuckDB <1.5.0 version cap and default to stable release instead
of nightly (spatial extension is native in 1.5+)
- Change default benchmark scale factor from 1 to 10 with 1800s query timeout
- Save benchmark results incrementally so partial results survive crashes
- Add continue-on-error to benchmark steps so artifacts are uploaded even
on failure, and the summarize job can report whatever completed
Closes #92
---
.github/workflows/benchmark.yml | 53 +++++++++++++++++++++++------------------
benchmark/run_benchmark.py | 10 +++++++-
2 files changed, 39 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 9294d55..faf384d 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -23,7 +23,7 @@ on:
scale_factor:
description: 'Scale factor for benchmark'
required: false
- default: '1'
+ default: '10'
type: choice
options:
- '0.1'
@@ -35,9 +35,9 @@ on:
default: 'duckdb,geopandas,sedonadb,spatial_polars'
type: string
timeout:
- description: 'Query timeout in seconds (default: 60, increase for full benchmark)'
+ description: 'Query timeout in seconds (default: 1800)'
required: false
- default: '60'
+ default: '1800'
type: string
sedonadb_version:
description: 'SedonaDB version (e.g., 1.0.0, leave empty for latest)'
@@ -76,7 +76,7 @@ on:
duckdb_nightly:
description: 'Use DuckDB pre-release/nightly build (ignores version if true)'
required: false
- default: true
+ default: false
type: boolean
concurrency:
@@ -85,9 +85,9 @@ concurrency:
env:
CARGO_TERM_COLOR: always
- SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '1' }}
+ SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '10' }}
BENCHMARK_ENGINES: ${{ github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars' }}
- QUERY_TIMEOUT: ${{ github.event.inputs.timeout || '60' }}
+ QUERY_TIMEOUT: ${{ github.event.inputs.timeout || '1800' }}
BENCHMARK_RUNS: ${{ github.event.inputs.runs || '3' }}
# Package versions (empty = latest, can be overridden via workflow_dispatch)
SEDONADB_VERSION: ${{ github.event.inputs.sedonadb_version }}
@@ -96,7 +96,7 @@ env:
SPATIAL_POLARS_VERSION: ${{ github.event.inputs.spatial_polars_version }}
# Nightly build options (default: true)
SEDONADB_NIGHTLY: ${{ github.event.inputs.sedonadb_nightly || 'true' }}
- DUCKDB_NIGHTLY: ${{ github.event.inputs.duckdb_nightly || 'true' }}
+ DUCKDB_NIGHTLY: ${{ github.event.inputs.duckdb_nightly || 'false' }}
# Hugging Face dataset for benchmark data
HF_DATASET: apache-sedona/spatialbench
HF_DATA_VERSION: v0.1.0
@@ -104,7 +104,7 @@ env:
jobs:
# Download benchmark data from Hugging Face
download-data:
- name: Download Data (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Download Data (SF${{ github.event.inputs.scale_factor || '10' }})
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -186,7 +186,7 @@ jobs:
du -sh benchmark-data-sf${{ env.SCALE_FACTOR }}/
benchmark-duckdb:
- name: Benchmark DuckDB (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Benchmark DuckDB (SF${{ github.event.inputs.scale_factor || '10' }})
needs: download-data
runs-on: ubuntu-latest
if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'duckdb')
@@ -212,9 +212,8 @@ jobs:
echo "DUCKDB_VERSION: ${{ env.DUCKDB_VERSION }}"
echo "======================================"
if [ "${{ env.DUCKDB_NIGHTLY }}" = "true" ]; then
- # Use --pre to install pre-release dev builds (e.g., 1.4.4.dev48)
- # Constraint <1.5.0 ensures we get 1.4.x branch dev builds
- pip install "duckdb<1.5.0" --pre pyarrow pandas
+ # Use --pre to install pre-release dev builds
+ pip install duckdb --pre pyarrow pandas
elif [ -n "${{ env.DUCKDB_VERSION }}" ]; then
pip install "duckdb==${{ env.DUCKDB_VERSION }}" pyarrow pandas
else
@@ -224,10 +223,12 @@ jobs:
- name: Pre-install DuckDB spatial extension
run: |
- # Dev builds don't have spatial extension in core_nightly, so always use default repo
- python -c "import duckdb; con = duckdb.connect(); con.execute('INSTALL spatial'); print('DuckDB spatial extension installed')"
+ # In DuckDB 1.5+, spatial is a native extension. INSTALL+LOAD still works for all versions.
+ python -c "import duckdb; con = duckdb.connect(); con.execute('INSTALL spatial'); con.execute('LOAD spatial'); print('DuckDB spatial extension installed and loaded')"
- name: Run DuckDB benchmark
+ id: run-benchmark
+ continue-on-error: true
run: |
python benchmark/run_benchmark.py \
--data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \
@@ -238,6 +239,7 @@ jobs:
--output duckdb_results.json
- name: Upload results
+ if: always() && hashFiles('duckdb_results.json') != ''
uses: actions/upload-artifact@v6
with:
name: duckdb-results-sf${{ env.SCALE_FACTOR }}
@@ -245,7 +247,7 @@ jobs:
retention-days: 30
benchmark-geopandas:
- name: Benchmark GeoPandas (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Benchmark GeoPandas (SF${{ github.event.inputs.scale_factor || '10' }})
needs: download-data
runs-on: ubuntu-latest
if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'geopandas')
@@ -274,6 +276,8 @@ jobs:
echo "Installed GeoPandas version: $(python -c 'from importlib.metadata import version; print(version("geopandas"))')"
- name: Run GeoPandas benchmark
+ id: run-benchmark
+ continue-on-error: true
run: |
python benchmark/run_benchmark.py \
--data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \
@@ -284,6 +288,7 @@ jobs:
--output geopandas_results.json
- name: Upload results
+ if: always() && hashFiles('geopandas_results.json') != ''
uses: actions/upload-artifact@v6
with:
name: geopandas-results-sf${{ env.SCALE_FACTOR }}
@@ -291,7 +296,7 @@ jobs:
retention-days: 30
benchmark-sedonadb:
- name: Benchmark SedonaDB (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Benchmark SedonaDB (SF${{ github.event.inputs.scale_factor || '10' }})
needs: download-data
runs-on: ubuntu-latest
if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'sedonadb')
@@ -330,6 +335,8 @@ jobs:
echo "Installed SedonaDB version: $(python -c 'from importlib.metadata import version; print(version("sedonadb"))')"
- name: Run SedonaDB benchmark
+ id: run-benchmark
+ continue-on-error: true
run: |
python benchmark/run_benchmark.py \
--data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \
@@ -340,6 +347,7 @@ jobs:
--output sedonadb_results.json
- name: Upload results
+ if: always() && hashFiles('sedonadb_results.json') != ''
uses: actions/upload-artifact@v6
with:
name: sedonadb-results-sf${{ env.SCALE_FACTOR }}
@@ -347,7 +355,7 @@ jobs:
retention-days: 30
benchmark-spatial-polars:
- name: Benchmark Spatial Polars (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Benchmark Spatial Polars (SF${{ github.event.inputs.scale_factor || '10' }})
needs: download-data
runs-on: ubuntu-latest
if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'spatial_polars')
@@ -376,6 +384,8 @@ jobs:
echo "Installed Spatial Polars version: $(python -c 'from importlib.metadata import version; print(version("spatial-polars"))')"
- name: Run Spatial Polars benchmark
+ id: run-benchmark
+ continue-on-error: true
run: |
python benchmark/run_benchmark.py \
--data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \
@@ -386,6 +396,7 @@ jobs:
--output spatial_polars_results.json
- name: Upload results
+ if: always() && hashFiles('spatial_polars_results.json') != ''
uses: actions/upload-artifact@v6
with:
name: spatial_polars-results-sf${{ env.SCALE_FACTOR }}
@@ -393,15 +404,14 @@ jobs:
retention-days: 30
summarize-results:
- name: Summarize Results (SF${{ github.event.inputs.scale_factor || '1' }})
+ name: Summarize Results (SF${{ github.event.inputs.scale_factor || '10' }})
needs: [benchmark-duckdb, benchmark-geopandas, benchmark-sedonadb, benchmark-spatial-polars]
- if: always() && (needs.benchmark-duckdb.result == 'success' || needs.benchmark-geopandas.result == 'success' || needs.benchmark-sedonadb.result == 'success' || needs.benchmark-spatial-polars.result == 'success')
+ if: always() && (needs.benchmark-duckdb.result != 'cancelled' || needs.benchmark-geopandas.result != 'cancelled' || needs.benchmark-sedonadb.result != 'cancelled' || needs.benchmark-spatial-polars.result != 'cancelled')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Download DuckDB results
- if: needs.benchmark-duckdb.result == 'success'
uses: actions/download-artifact@v7
with:
name: duckdb-results-sf${{ env.SCALE_FACTOR }}
@@ -409,7 +419,6 @@ jobs:
continue-on-error: true
- name: Download GeoPandas results
- if: needs.benchmark-geopandas.result == 'success'
uses: actions/download-artifact@v7
with:
name: geopandas-results-sf${{ env.SCALE_FACTOR }}
@@ -417,7 +426,6 @@ jobs:
continue-on-error: true
- name: Download SedonaDB results
- if: needs.benchmark-sedonadb.result == 'success'
uses: actions/download-artifact@v7
with:
name: sedonadb-results-sf${{ env.SCALE_FACTOR }}
@@ -425,7 +433,6 @@ jobs:
continue-on-error: true
- name: Download Spatial Polars results
- if: needs.benchmark-spatial-polars.result == 'success'
uses: actions/download-artifact@v7
with:
name: spatial_polars-results-sf${{ env.SCALE_FACTOR }}
diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py
index e05e237..4a8d8d5 100755
--- a/benchmark/run_benchmark.py
+++ b/benchmark/run_benchmark.py
@@ -428,6 +428,7 @@ def run_benchmark(
timeout: int,
scale_factor: float,
runs: int = 3,
+ output_file: str | None = None,
) -> BenchmarkSuite:
"""Generic benchmark runner for any engine.
@@ -438,6 +439,9 @@ def run_benchmark(
If runs > 1 and the first run succeeds, additional runs are performed
and the average time is reported for fair comparison.
+
+ If output_file is provided, results are saved incrementally after each
+ query so that partial results survive if the runner crashes mid-way.
"""
from importlib.metadata import version as pkg_version
@@ -538,6 +542,10 @@ def run_benchmark(
if result.status == "success":
suite.total_time += result.time_seconds
+ # Save partial results after each query so they survive crashes
+ if output_file:
+ save_results([suite], output_file)
+
return suite
@@ -629,7 +637,7 @@ def main():
print(f" {table}: {path}")
results = [
- run_benchmark(engine, data_paths, queries, args.timeout, args.scale_factor, args.runs)
+ run_benchmark(engine, data_paths, queries, args.timeout, args.scale_factor, args.runs, args.output)
for engine in engines
]