You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ja...@apache.org on 2026/08/13 17:36:40 UTC
(sedona-db) 01/01: test(python): opt-in SedonaDB vs Sedona Spark raster parity harness
This is an automated email from the ASF dual-hosted git repository.
james-willis pushed a commit to branch jw/sedona-spark-parity
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
commit 7ae7c5b7d061609f2d0735e728c895d5cb308426
Author: jameswillis <ja...@wherobots.com>
AuthorDate: Thu Aug 13 10:36:23 2026 -0700
test(python): opt-in SedonaDB vs Sedona Spark raster parity harness
---
python/sedonadb/python/sedonadb/testing.py | 36 +++++
python/sedonadb/python/sedonadb/testing_spark.py | 176 +++++++++++++++++++++
.../functions/test_rs_raster_out_spark_parity.py | 69 ++++++++
.../tests/functions/test_rs_scalar_spark_parity.py | 76 +++++++++
4 files changed, 357 insertions(+)
diff --git a/python/sedonadb/python/sedonadb/testing.py b/python/sedonadb/python/sedonadb/testing.py
index 5053bf094..88d212a7e 100644
--- a/python/sedonadb/python/sedonadb/testing.py
+++ b/python/sedonadb/python/sedonadb/testing.py
@@ -176,6 +176,26 @@ class DBEngine:
"""Copy an Arrow readable into an engine's native table format"""
raise NotImplementedError()
+ def create_raster_view(self, name, path) -> "DBEngine":
+ """Register a one-row view `name` with a raster column `rast` read from
+ the GeoTIFF at `path`.
+
+ The single dialect-specific shim in the raster parity suite: each engine
+ ingests the file its own way (SedonaDB via `RS_FromPath`, Sedona Spark
+ via `RS_FromGeoTiff` over a `binaryFile` read), so the query under test
+ stays one shared SQL string that references `rast`.
+ """
+ raise NotImplementedError()
+
+ def decode_raster_result(self, sql):
+ """Run `sql` (which selects a single raster column) and decode the result
+ raster to a `sedonadb.raster_testing.DecodedRaster` — pixels, geotransform,
+ and per-band nodata — for cross-engine comparison. `None` if the raster is
+ NULL. Each engine decodes its own way (SedonaDB reads the native raster
+ column; Sedona Spark transports it out as GeoTIFF bytes).
+ """
+ raise NotImplementedError()
+
def execute_and_collect(self, query):
"""Execute a query and collect results to the driver
@@ -375,6 +395,22 @@ class SedonaDB(DBEngine):
self.con.create_data_frame(obj).to_view(name, overwrite=True)
return self
+ def create_raster_view(self, name, path) -> "SedonaDB":
+ self.con.sql(f"SELECT RS_FromPath('{path}') AS rast").to_view(
+ name, overwrite=True
+ )
+ return self
+
+ def decode_raster_result(self, sql):
+ from sedonadb.raster_testing import decode_raster
+
+ # RS_FromPath yields an OutDb (lazy) raster, so materialize the result
+ # before reading its pixels; Sedona Spark's RS_FromGeoTiff loads eagerly,
+ # so this keeps the two engines comparing the same bytes.
+ loaded = f"SELECT RS_EnsureLoaded(c) AS c FROM ({sql}) AS sub(c)"
+ table = self.execute_and_collect(loaded)
+ return decode_raster(table.column(0)[0])
+
def execute_and_collect(self, query) -> "sedonadb.dataframe.DataFrame":
# Use to_arrow_table() to maintain ordering of the input table
return self.con.sql(query).to_arrow_table()
diff --git a/python/sedonadb/python/sedonadb/testing_spark.py b/python/sedonadb/python/sedonadb/testing_spark.py
new file mode 100644
index 000000000..540d84d5e
--- /dev/null
+++ b/python/sedonadb/python/sedonadb/testing_spark.py
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Sedona Spark as a :class:`~sedonadb.testing.DBEngine`.
+
+The compatibility target for SedonaDB's SQL surface is Sedona Spark, so parity
+tests broadcast one shared SQL string to both engines and compare strictly —
+the same pattern the geometry suite uses for SedonaDB vs PostGIS. This lives in
+its own module because the engine needs pyspark, a JVM, and network access,
+none of which the core :mod:`sedonadb.testing` module should pull in.
+
+Bootstrapping a ``SparkSession`` (downloading the Sedona jars from Maven, JVM
+startup) costs tens of seconds, so these tests are opt-in: :meth:`create_or_skip`
+skips — even under ``SEDONADB_PYTHON_NO_SKIP_TESTS`` — unless
+``SEDONADB_RUN_SPARK_TESTS`` is set, in which case construction failures
+propagate.
+
+Requires Spark 4.0+ (results travel out of the JVM through ``DataFrame.toArrow``,
+which preserves nulls Arrow-natively — unlike ``toPandas``, which would coerce a
+nodata ``None`` to ``NaN`` and mask exactly the value under test).
+"""
+
+import os
+
+import pyarrow as pa
+
+from sedonadb.testing import DBEngine
+
+# Sedona jar coordinates. Override with SEDONADB_SEDONA_SPARK_PACKAGES (full
+# Maven coordinates) when testing against a different Sedona release.
+SEDONA_SPARK_VERSION = "1.9.0"
+GEOTOOLS_WRAPPER_VERSION = "1.9.0-33.5"
+
+
+class SedonaSpark(DBEngine):
+ """Runs Sedona Spark SQL — the compatibility-target dialect.
+
+ One local ``SparkSession`` is bootstrapped per process and shared across
+ engine instances. Rasters are read from GeoTIFF files with
+ ``RS_FromGeoTiff`` over a ``binaryFile`` scan (see :meth:`create_raster_view`).
+ """
+
+ _spark = None
+
+ def __init__(self):
+ self._session = self._ensure_session()
+
+ @classmethod
+ def name(cls) -> str:
+ return "sedona-spark"
+
+ @classmethod
+ def install_hint(cls) -> str:
+ return (
+ "- Run `pip install 'pyspark>=4.0' apache-sedona` (needs a JVM; the "
+ "first run downloads the Sedona jars from Maven)\n"
+ "- Set SEDONADB_RUN_SPARK_TESTS=true to opt in"
+ )
+
+ @classmethod
+ def create_or_skip(cls, *args, **kwargs) -> "SedonaSpark":
+ import pytest
+
+ if os.environ.get("SEDONADB_RUN_SPARK_TESTS", "false") not in ("true", "1"):
+ pytest.skip("Sedona Spark parity tests are opt-in:\n" + cls.install_hint())
+ return cls(*args, **kwargs)
+
+ @classmethod
+ def _ensure_session(cls):
+ if SedonaSpark._spark is None:
+ from sedona.spark import SedonaContext
+
+ config = (
+ SedonaContext.builder()
+ .master("local[2]")
+ .appName("sedonadb-spark-parity")
+ .config("spark.jars.packages", cls._packages())
+ .config("spark.jars.ivy", cls._ivy_dir())
+ .config("spark.ui.enabled", "false")
+ .getOrCreate()
+ )
+ SedonaSpark._spark = SedonaContext.create(config)
+ return SedonaSpark._spark
+
+ @staticmethod
+ def _packages() -> str:
+ env = os.environ.get("SEDONADB_SEDONA_SPARK_PACKAGES")
+ if env:
+ return env
+ import pyspark
+
+ major, minor = (int(part) for part in pyspark.__version__.split(".")[:2])
+ # Sedona publishes per-Spark-minor artifacts. A pyspark older than every
+ # published artifact is an error (a newer jar on an older runtime fails
+ # at class load); a pyspark newer than the newest artifact tries the
+ # newest, which usually loads — override the coordinates if it doesn't.
+ known = ("3.5", "4.0")
+ spark_suffix = f"{major}.{minor}"
+ if spark_suffix not in known:
+ if (major, minor) < (3, 5):
+ raise RuntimeError(
+ f"No Sedona {SEDONA_SPARK_VERSION} artifact supports pyspark "
+ f"{pyspark.__version__}; install pyspark >= 3.5 or set "
+ "SEDONADB_SEDONA_SPARK_PACKAGES to explicit Maven coordinates"
+ )
+ spark_suffix = known[-1]
+ scala_suffix = "2.13" if spark_suffix == "4.0" else "2.12"
+ return (
+ f"org.apache.sedona:sedona-spark-shaded-{spark_suffix}_{scala_suffix}:"
+ f"{SEDONA_SPARK_VERSION},"
+ f"org.datasyslab:geotools-wrapper:{GEOTOOLS_WRAPPER_VERSION}"
+ )
+
+ @staticmethod
+ def _ivy_dir() -> str:
+ """Directory Ivy resolves ``spark.jars.packages`` into.
+
+ Pinned so CI can cache the downloaded jars: newer Ivy releases (bundled
+ with newer Spark) moved the default location, silently breaking a cache
+ keyed on the old path. Override with SEDONADB_SPARK_IVY_DIR.
+ """
+ return os.environ.get(
+ "SEDONADB_SPARK_IVY_DIR",
+ os.path.join(os.path.expanduser("~"), ".ivy2"),
+ )
+
+ def create_raster_view(self, name, path) -> "SedonaSpark":
+ self._session.read.format("binaryFile").load(str(path)).selectExpr(
+ "RS_FromGeoTiff(content) AS rast"
+ ).createOrReplaceTempView(name)
+ return self
+
+ def execute_and_collect(self, query):
+ return self._session.sql(query)
+
+ def result_to_table(self, result) -> pa.Table:
+ # toArrow() preserves nulls Arrow-natively; toPandas() would turn a
+ # nodata None into a float NaN and mask the value under test.
+ return result.toArrow()
+
+ def decode_raster_result(self, sql):
+ from sedonadb.raster_testing import DecodedRaster, decode_geotiff_bytes
+
+ # A raster can't leave the JVM as a native column, so transport it as
+ # GeoTIFF bytes. geotools' writer refuses a CRS-less coverage, so stamp
+ # an arbitrary SRID first — transport-only, it doesn't touch pixels or
+ # the geotransform (and DecodedRaster carries no CRS to compare). Nodata
+ # is read separately through RS_BandNoDataValue so the comparison sees
+ # the engine's own claim rather than the GeoTIFF writer's encoding of it.
+ result = self._session.sql(sql).toDF("r").cache()
+ try:
+ head = result.selectExpr(
+ "RS_AsGeoTiff(RS_SetSRID(r, 3857)) AS t", "RS_NumBands(r) AS n"
+ ).first()
+ if head is None or head.t is None:
+ return None
+ decoded = decode_geotiff_bytes(bytes(head.t))
+ nodata = result.selectExpr(
+ *[f"RS_BandNoDataValue(r, {b}) AS nd{b}" for b in range(1, head.n + 1)]
+ ).first()
+ return DecodedRaster(decoded.pixels, decoded.gdal_transform, list(nodata))
+ finally:
+ result.unpersist()
diff --git a/python/sedonadb/tests/functions/test_rs_raster_out_spark_parity.py b/python/sedonadb/tests/functions/test_rs_raster_out_spark_parity.py
new file mode 100644
index 000000000..90965f0bb
--- /dev/null
+++ b/python/sedonadb/tests/functions/test_rs_raster_out_spark_parity.py
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""SedonaDB vs Sedona Spark parity for raster-in / raster-out functions.
+
+These exercise the raster round-trip *out* of each engine: SedonaDB decodes its
+native raster column; Sedona Spark transports the result as GeoTIFF bytes
+(`RS_AsGeoTiff`) and decodes it with rasterio. Both sides land as a
+`DecodedRaster` (pixels + geotransform + per-band nodata) and are compared with
+`assert_decoded_equal`. Same opt-in (`SEDONADB_RUN_SPARK_TESTS`) and
+`xfail`-for-known-divergence policy as the scalar suite.
+
+`RS_SetBandNoDataValue` is the first case on purpose: it is raster-in/raster-out
+but passes pixels through untouched, so a mismatch is a round-trip bug, not an
+operation divergence — it isolates the transport. Pixel-transforming ops
+(RS_Resample, RS_MapAlgebra) come next, with their known divergences as xfails.
+"""
+
+import pytest
+
+from sedonadb.raster_testing import (
+ assert_decoded_equal,
+ random_raster_data,
+ write_geotiff,
+)
+from sedonadb.testing import SedonaDB
+from sedonadb.testing_spark import SedonaSpark
+
+pytest.importorskip("rasterio") # write_geotiff / GeoTIFF decode go through rasterio
+
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 2, 6, 7
+
+# Each value is representable in its dtype, so it packs into the band exactly.
+BAND_NODATA = {"uint8": 200.0, "int32": -99999.0, "float64": -12345.5}
+
+
+@pytest.mark.parametrize("dtype", list(BAND_NODATA))
+def test_rs_setbandnodata_raster_out_spark_parity(dtype, tmp_path):
+ """RS_SetBandNoDataValue sets band 1's nodata and passes pixels through, so
+ SedonaDB and Sedona Spark must return the same raster: identical pixels and
+ geotransform, band 1's nodata set, band 2's still absent."""
+ tif = tmp_path / f"src_{dtype}.tif"
+ data = random_raster_data(dtype, bands=BANDS, height=HEIGHT, width=WIDTH)
+ write_geotiff(tif, data, gdal_transform=GDAL_TRANSFORM) # no nodata to start
+
+ sedona = SedonaDB.create_or_skip()
+ spark = SedonaSpark.create_or_skip() # skips the whole test unless opted in
+ for eng in (sedona, spark):
+ eng.create_raster_view("src", tif)
+
+ sql = f"SELECT RS_SetBandNoDataValue(rast, 1, {BAND_NODATA[dtype]}) FROM src"
+ assert_decoded_equal(
+ sedona.decode_raster_result(sql),
+ spark.decode_raster_result(sql),
+ )
diff --git a/python/sedonadb/tests/functions/test_rs_scalar_spark_parity.py b/python/sedonadb/tests/functions/test_rs_scalar_spark_parity.py
new file mode 100644
index 000000000..cdd0e8dc2
--- /dev/null
+++ b/python/sedonadb/tests/functions/test_rs_scalar_spark_parity.py
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""SedonaDB vs Sedona Spark parity for the scalar raster readers.
+
+Unlike the geometry suite (which pins each engine against a fixed expected
+value), these tests assert the two engines return the *same* result for the
+*same* SQL over the *same* GeoTIFF — the parity question directly, with no
+oracle. SedonaDB's own correctness is covered by the rasterio-oracle tests in
+`test_rs_value.py`; this suite only asks whether Sedona Spark agrees.
+
+The whole test skips unless `SEDONADB_RUN_SPARK_TESTS` is set (see
+`SedonaSpark.create_or_skip`). Where the two engines are known to diverge and we
+intend to close the gap, mark the case `xfail(reason=...)` so the suite doubles
+as a catalog of what to fix — it flips to xpass the day the fix lands.
+"""
+
+import pytest
+
+from sedonadb.raster_testing import random_raster_data, write_geotiff
+from sedonadb.testing import SedonaDB
+from sedonadb.testing_spark import SedonaSpark
+
+pytest.importorskip("rasterio") # write_geotiff writes the fixtures via rasterio
+
+# North-up, CRS-less: origin (100, 500) with 2-wide by 3-tall pixels. These
+# functions take no geometry, so a CRS would only add a reprojection difference.
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 2, 6, 7
+
+# Each value is representable in its dtype, so the nodata reads back exactly.
+BAND_NODATA = {"uint8": 200.0, "int32": -99999.0, "float64": -12345.5}
+
+
+def _one(engine, sql):
+ """The single scalar (or list) result of `sql` on `engine`, as a Python value."""
+ table = engine.result_to_table(engine.execute_and_collect(sql))
+ return table.column(0)[0].as_py()
+
+
+@pytest.mark.parametrize("dtype", list(BAND_NODATA))
+def test_rs_band_nodata_spark_parity(dtype, tmp_path):
+ """SedonaDB and Sedona Spark read back the same band nodata for the same
+ GeoTIFF, and both return NULL for a band written without one."""
+ data = random_raster_data(dtype, bands=BANDS, height=HEIGHT, width=WIDTH)
+ with_nodata = tmp_path / f"nd_{dtype}.tif"
+ without_nodata = tmp_path / f"nond_{dtype}.tif"
+ write_geotiff(
+ with_nodata, data, gdal_transform=GDAL_TRANSFORM, nodata=BAND_NODATA[dtype]
+ )
+ write_geotiff(without_nodata, data, gdal_transform=GDAL_TRANSFORM)
+
+ sedona = SedonaDB.create_or_skip()
+ spark = SedonaSpark.create_or_skip() # skips the whole test unless opted in
+ for eng in (sedona, spark):
+ eng.create_raster_view("nd_raster", with_nodata)
+ eng.create_raster_view("nond_raster", without_nodata)
+
+ for band in (1, 2):
+ with_sql = f"SELECT RS_BandNoDataValue(rast, {band}) FROM nd_raster"
+ without_sql = f"SELECT RS_BandNoDataValue(rast, {band}) FROM nond_raster"
+ assert _one(sedona, with_sql) == _one(spark, with_sql)
+ assert _one(sedona, without_sql) == _one(spark, without_sql)