You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2026/05/01 06:59:15 UTC
(sedona) branch master updated: [GH-2700] Refresh docker example notebooks: scaffolding + quickstart (#2876)
This is an automated email from the ASF dual-hosted git repository.
jiayuasu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 78c0c90f54 [GH-2700] Refresh docker example notebooks: scaffolding + quickstart (#2876)
78c0c90f54 is described below
commit 78c0c90f547cb1841a7c1e388c72c16f2a304642
Author: Jia Yu <ji...@apache.org>
AuthorDate: Thu Apr 30 23:59:09 2026 -0700
[GH-2700] Refresh docker example notebooks: scaffolding + quickstart (#2876)
---
docker/test-notebooks.sh | 37 ++++-
docs/tutorial/sql.md | 4 +-
docs/usecases/00-quickstart.ipynb | 170 +++++++++++++++++++++
docs/usecases/{ => legacy}/ApacheSedonaCore.ipynb | 0
.../usecases/{ => legacy}/ApacheSedonaRaster.ipynb | 0
docs/usecases/{ => legacy}/ApacheSedonaSQL.ipynb | 0
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 0
.../Sedona_OvertureMaps_GeoParquet.ipynb | 0
8 files changed, 202 insertions(+), 9 deletions(-)
diff --git a/docker/test-notebooks.sh b/docker/test-notebooks.sh
index e119466115..c23095738d 100755
--- a/docker/test-notebooks.sh
+++ b/docker/test-notebooks.sh
@@ -18,6 +18,11 @@
# under the License.
set -e
+# Without pipefail, a failing `python3 ... | tee ...` pipeline returns tee's
+# (zero) exit status, so a notebook crash or timeout would be silently
+# misreported as a pass. Enable pipefail so the per-notebook `if` block
+# below sees the real exit code.
+set -o pipefail
# Set up environment variables if not already set
export SPARK_HOME=${SPARK_HOME:-/opt/spark}
@@ -60,11 +65,26 @@ NOTEBOOK_COUNT=$(echo "$NOTEBOOK_FILES" | wc -l)
echo "Found $NOTEBOOK_COUNT notebook file(s) to test (root level only, excluding subdirectories)"
echo ""
+SKIPPED_TESTS=0
+
# Convert and test each notebook
for NOTEBOOK in $NOTEBOOK_FILES; do
NOTEBOOK_NAME=$(basename "$NOTEBOOK" .ipynb)
PYTHON_FILE="${NOTEBOOK%.ipynb}.py"
+ # A notebook can declare a network dependency by including the marker
+ # requires-network: true
+ # anywhere in its source (typically in the first markdown cell). When
+ # SEDONA_NOTEBOOK_OFFLINE=1 is set, such notebooks are skipped so the
+ # harness still passes in sandboxed CI environments without outbound
+ # network access.
+ if [ "${SEDONA_NOTEBOOK_OFFLINE:-0}" = "1" ] && grep -q "requires-network: true" "$NOTEBOOK"; then
+ echo "Skipping (offline mode): $NOTEBOOK_NAME"
+ echo ""
+ SKIPPED_TESTS=$((SKIPPED_TESTS + 1))
+ continue
+ fi
+
echo "Testing: $NOTEBOOK_NAME"
echo " Converting notebook to Python..."
@@ -179,13 +199,15 @@ PYTHON_CLEANUP
continue
fi
- # Run the full Python script with timeout (600 seconds = 10 minutes)
- echo " Running Python script (600 second timeout)..."
+ # Run the full Python script with timeout (900 seconds = 15 minutes).
+ # Bumped from 600s to absorb network variance in notebooks that hit STAC
+ # or remote object stores. Local-data notebooks finish well under 60s.
+ echo " Running Python script (900 second timeout)..."
cd "$EXAMPLES_DIR/.."
# Use timeout with progress reporting
START_TIME=$(date +%s)
- if timeout 600 python3 "$PYTHON_FILE" | tee /tmp/notebook_output_$$.log; then
+ if timeout 900 python3 "$PYTHON_FILE" | tee /tmp/notebook_output_$$.log; then
END_TIME=$(date +%s)
ELAPSED=$((END_TIME - START_TIME))
echo " ✓ Test passed (completed in ${ELAPSED}s)"
@@ -195,7 +217,7 @@ PYTHON_CLEANUP
END_TIME=$(date +%s)
ELAPSED=$((END_TIME - START_TIME))
if [ $EXIT_CODE -eq 124 ]; then
- echo " ✗ Test timed out (exceeded 600 seconds, ran for ${ELAPSED}s)"
+ echo " ✗ Test timed out (exceeded 900 seconds, ran for ${ELAPSED}s)"
echo " Last 20 lines of output:"
tail -20 /tmp/notebook_output_$$.log 2>/dev/null || echo " (no output captured)"
else
@@ -213,9 +235,10 @@ done
echo "========================================="
echo "Test Summary"
echo "========================================="
-echo "Passed: $PASSED_TESTS"
-echo "Failed: $FAILED_TESTS"
-echo "Total: $((PASSED_TESTS + FAILED_TESTS))"
+echo "Passed: $PASSED_TESTS"
+echo "Failed: $FAILED_TESTS"
+echo "Skipped: $SKIPPED_TESTS"
+echo "Total: $((PASSED_TESTS + FAILED_TESTS + SKIPPED_TESTS))"
echo "========================================="
if [ $FAILED_TESTS -gt 0 ]; then
diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md
index fdf6d63ad4..e4371f64b6 100644
--- a/docs/tutorial/sql.md
+++ b/docs/tutorial/sql.md
@@ -855,7 +855,7 @@ SedonaPyDeck.create_choropleth_map(df=groupedresult, plot_col="AirportCount")
The dataset used is available [here](https://github.com/apache/sedona/tree/b66e768155866a38ba2e3404f1151cac14fad5ea/docs/usecases/data/ne_50m_airports) and
-can also be found in the example notebook available [here](https://github.com/apache/sedona/blob/master/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb)
+can also be found in the example notebook available [here](https://github.com/apache/sedona/blob/master/docs/usecases/legacy/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb)
#### Creating a Geometry map using SedonaPyDeck
@@ -923,7 +923,7 @@ SedonaKepler.create_map(df=groupedresult, name="AirportCount")
The dataset used is available [here](https://github.com/apache/sedona/tree/b66e768155866a38ba2e3404f1151cac14fad5ea/docs/usecases/data/ne_50m_airports) and
-can also be found in the example notebook available [here](https://github.com/apache/sedona/blob/master/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb)
+can also be found in the example notebook available [here](https://github.com/apache/sedona/blob/master/docs/usecases/legacy/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb)
Details on all the APIs available by SedonaKepler are listed in the [SedonaKepler API docs](../api/sql/Visualization-SedonaKepler.md)
diff --git a/docs/usecases/00-quickstart.ipynb b/docs/usecases/00-quickstart.ipynb
new file mode 100644
index 0000000000..91d75bfe74
--- /dev/null
+++ b/docs/usecases/00-quickstart.ipynb
@@ -0,0 +1,170 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": "<!--\n Licensed to the Apache Software Foundation (ASF) under one\n or more contributor license agreements. See the NOTICE file\n distributed with this work for additional information\n regarding copyright ownership. The ASF licenses this file\n to you under the Apache License, Version 2.0 (the\n \"License\"); you may not use this file except in compliance\n with the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unl [...]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sedona.spark import SedonaContext\n",
+ "\n",
+ "config = SedonaContext.builder().master(\"spark://localhost:7077\").getOrCreate()\n",
+ "sedona = SedonaContext.create(config)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read country polygons. The shapefile reader is a first-class Spark V2 datasource\n",
+ "# in Sedona; no manual binary parsing.\n",
+ "countries = (\n",
+ " sedona.read.format(\"shapefile\")\n",
+ " .load(\"data/ne_50m_admin_0_countries_lakes\")\n",
+ " .selectExpr(\n",
+ " \"geometry\",\n",
+ " \"NAME as country\",\n",
+ " \"CONTINENT as continent\",\n",
+ " \"POP_EST as population\",\n",
+ " )\n",
+ ")\n",
+ "countries.show(5, truncate=40)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "airports = (\n",
+ " sedona.read.format(\"shapefile\")\n",
+ " .load(\"data/ne_50m_airports\")\n",
+ " .selectExpr(\n",
+ " \"geometry\",\n",
+ " \"name as airport\",\n",
+ " \"type as airport_type\",\n",
+ " )\n",
+ ")\n",
+ "airports.show(5, truncate=40)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Spatial join: which country contains each airport?\n",
+ "# Sedona inserts a broadcast spatial index when one side is small.\n",
+ "airports.createOrReplaceTempView(\"airports\")\n",
+ "countries.createOrReplaceTempView(\"countries\")\n",
+ "\n",
+ "joined = sedona.sql(\"\"\"\n",
+ " SELECT c.country, c.continent, c.population, a.airport\n",
+ " FROM countries c\n",
+ " JOIN airports a ON ST_Contains(c.geometry, a.geometry)\n",
+ "\"\"\")\n",
+ "joined.show(5, truncate=40)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Rank countries by airports per million people.\n",
+ "joined.createOrReplaceTempView(\"joined\")\n",
+ "\n",
+ "ranked = sedona.sql(\"\"\"\n",
+ " SELECT\n",
+ " country,\n",
+ " continent,\n",
+ " population,\n",
+ " COUNT(*) AS airport_count,\n",
+ " ROUND(COUNT(*) * 1e6 / NULLIF(population, 0), 2) AS airports_per_million\n",
+ " FROM joined\n",
+ " WHERE population > 1000000\n",
+ " GROUP BY country, continent, population\n",
+ " ORDER BY airports_per_million DESC\n",
+ "\"\"\")\n",
+ "ranked.show(10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Attach the rank back onto country geometries and write GeoParquet 1.1.\n",
+ "# Sedona auto-populates the covering-bbox metadata and projjson CRS from the\n",
+ "# geometry SRID, so downstream readers can prune partitions efficiently.\n",
+ "import os\n",
+ "import shutil\n",
+ "\n",
+ "ranked.createOrReplaceTempView(\"ranked\")\n",
+ "ranked_geo = sedona.sql(\"\"\"\n",
+ " SELECT c.country, c.continent,\n",
+ " r.airport_count, r.airports_per_million,\n",
+ " c.geometry\n",
+ " FROM ranked r\n",
+ " JOIN countries c USING (country)\n",
+ "\"\"\")\n",
+ "\n",
+ "out_path = \"/tmp/airports_per_country.parquet\"\n",
+ "if os.path.exists(out_path):\n",
+ " shutil.rmtree(out_path)\n",
+ "ranked_geo.coalesce(1).write.format(\"geoparquet\").save(out_path)\n",
+ "\n",
+ "sedona.read.format(\"geoparquet\").load(out_path).show(5, truncate=40)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sedona.spark import SedonaKepler\n",
+ "\n",
+ "ranked_geo_df = sedona.read.format(\"geoparquet\").load(out_path)\n",
+ "SedonaKepler.create_map(df=ranked_geo_df, name=\"airports_per_million\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": "## What just happened?\n\nIn a handful of cells we ran an end-to-end spatial workflow:\n\n1. **Read** two shapefiles directly into Spark DataFrames via Sedona's V2 shapefile datasource.\n2. **Joined** them spatially with `ST_Contains` — Sedona inserts a broadcast spatial index automatically when one side is small.\n3. **Aggregated and ranked** countries by airports per capita.\n4. **Wrote** the result as **GeoParquet 1.1**; covering-bbox metadata and projjson CRS were popul [...]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/docs/usecases/ApacheSedonaCore.ipynb b/docs/usecases/legacy/ApacheSedonaCore.ipynb
similarity index 100%
rename from docs/usecases/ApacheSedonaCore.ipynb
rename to docs/usecases/legacy/ApacheSedonaCore.ipynb
diff --git a/docs/usecases/ApacheSedonaRaster.ipynb b/docs/usecases/legacy/ApacheSedonaRaster.ipynb
similarity index 100%
rename from docs/usecases/ApacheSedonaRaster.ipynb
rename to docs/usecases/legacy/ApacheSedonaRaster.ipynb
diff --git a/docs/usecases/ApacheSedonaSQL.ipynb b/docs/usecases/legacy/ApacheSedonaSQL.ipynb
similarity index 100%
rename from docs/usecases/ApacheSedonaSQL.ipynb
rename to docs/usecases/legacy/ApacheSedonaSQL.ipynb
diff --git a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb b/docs/usecases/legacy/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
similarity index 100%
rename from docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
rename to docs/usecases/legacy/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
diff --git a/docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb b/docs/usecases/legacy/Sedona_OvertureMaps_GeoParquet.ipynb
similarity index 100%
rename from docs/usecases/Sedona_OvertureMaps_GeoParquet.ipynb
rename to docs/usecases/legacy/Sedona_OvertureMaps_GeoParquet.ipynb