You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/06/16 21:00:49 UTC
(sedona-db) branch main updated: feat(sedonadb/python): Add lazy raster creator for testing (#966)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new 65fe1287a feat(sedonadb/python): Add lazy raster creator for testing (#966)
65fe1287a is described below
commit 65fe1287ad319f59d8af5ef03ed5f0edec57b732
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Tue Jun 16 16:00:44 2026 -0500
feat(sedonadb/python): Add lazy raster creator for testing (#966)
---
python/sedonadb/python/sedonadb/raster.py | 92 ++++++++++++++++++++++++++++++-
python/sedonadb/src/lib.rs | 1 +
python/sedonadb/src/schema.rs | 7 +++
python/sedonadb/tests/test_raster.py | 49 ++++++++++++++++
4 files changed, 147 insertions(+), 2 deletions(-)
diff --git a/python/sedonadb/python/sedonadb/raster.py b/python/sedonadb/python/sedonadb/raster.py
index 369017626..884b39cdd 100644
--- a/python/sedonadb/python/sedonadb/raster.py
+++ b/python/sedonadb/python/sedonadb/raster.py
@@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-from typing import List, Optional, TYPE_CHECKING, Tuple
+from typing import List, Optional, TYPE_CHECKING, Tuple, Any, Iterable
import geoarrow.types as gat
import pyarrow as pa
+from sedonadb._lib import raster_type
+
if TYPE_CHECKING:
import numpy as np
@@ -38,6 +40,8 @@ BAND_DATA_TYPES = {
10: "Int8",
}
+BAND_DATA_TYPE_IDS = {v.lower(): k for k, v in BAND_DATA_TYPES.items()}
+
# Python struct module format characters for band data types
BAND_DATA_TYPE_STRUCT_CHARS = {
1: "B",
@@ -56,6 +60,80 @@ BAND_DATA_TYPE_STRUCT_CHARS = {
class Raster:
"""Python representation of a sedona.raster scalar value."""
+ @staticmethod
+ def lazy(
+ uri: str,
+ shape: Iterable[int],
+ dtype: str,
+ *,
+ format: Optional[str] = None,
+ crs: Any = None,
+ ) -> "Raster":
+ """Create a lazy raster that references external data without loading it.
+
+ This creates a raster with a single band that points to an external data
+ source (e.g., a file on disk or cloud storage) without reading the actual
+ pixel data into memory.
+
+ Args:
+ uri: The URI of the external data source (e.g., file path or cloud URL).
+ This URI is not validated unless the pixels are read.
+ shape: The shape of the raster as (height, width). Must have exactly
+ two dimensions.
+ dtype: The pixel data type (e.g., 'uint8', 'float32', 'int16').
+ format: The format of the external data (e.g., 'tif', or 'zarr'). If None,
+ the format will be inferred when the data is accessed.
+ crs: The coordinate reference system. Can be any value accepted by
+ geoarrow.types.type_spec (e.g., string, pyproj.CRS).
+
+ Returns:
+ A new Raster instance with a single band referencing the external data.
+
+ Raises:
+ ValueError: If shape does not have exactly two dimensions.
+
+ Examples:
+ >>> raster = Raster.lazy("s3://bucket/image.tif", (1024, 2048), "uint8")
+ """
+ shape = list(shape)
+ if len(shape) != 2:
+ raise ValueError("lazy() currently supports exactly two dimensions")
+
+ if crs is not None:
+ crs = gat.type_spec(crs=crs).crs.to_json()
+
+ dtype = dtype.lower()
+ if dtype not in BAND_DATA_TYPE_IDS:
+ raise ValueError(f"Unsupported raster dtype: {dtype}")
+
+ # Create the band
+ band = {
+ "name": None,
+ "dim_names": ["y", "x"],
+ "source_shape": shape,
+ "data_type": BAND_DATA_TYPE_IDS[dtype],
+ "nodata": None,
+ "view": None,
+ "outdb_uri": uri,
+ "outdb_format": format,
+ "data": b"",
+ }
+
+ # Create the raster
+ raster = {
+ "crs": crs,
+ "transform": [0.0, 1.0, 0.0, 0.0, 0.0, -1.0],
+ "spatial_dims": ["x", "y"],
+ "spatial_shape": [shape[1], shape[0]],
+ "bands": [band],
+ }
+
+ storage_type = pa.DataType._import_from_c_capsule(
+ raster_type().__arrow_c_schema__()
+ )
+ array = pa.array([raster], type=storage_type)
+ return Raster(array)
+
def __init__(self, array, i=0):
"""Create a Raster from an Arrow array at index i."""
if isinstance(array, pa.ExtensionArray):
@@ -198,7 +276,7 @@ class RasterType(pa.ExtensionType):
This extension type wraps a struct storage type representing raster data.
"""
- def __init__(self, storage_type: pa.DataType):
+ def __init__(self, storage_type: Any = None):
"""Create a RasterType with the given storage type.
Parameters
@@ -206,6 +284,9 @@ class RasterType(pa.ExtensionType):
storage_type : pa.DataType
The underlying Arrow storage type (must be a struct type).
"""
+ if storage_type is None:
+ storage_type = RASTER_STORAGE_TYPE
+
if not pa.types.is_struct(storage_type):
raise TypeError(f"storage_type must be a struct type, not {storage_type}")
super().__init__(storage_type, EXTENSION_NAME)
@@ -241,3 +322,10 @@ def register_extension_type():
except pa.ArrowKeyError:
# Already registered
pass
+
+
+# The storage type for a raster. To ensure we get this exactly right,
+# import from Rust via the Arrow PyCapsule interface.
+RASTER_STORAGE_TYPE = pa.DataType._import_from_c_capsule(
+ raster_type().__arrow_c_schema__()
+)
diff --git a/python/sedonadb/src/lib.rs b/python/sedonadb/src/lib.rs
index 0d1d51818..ad2b82a8c 100644
--- a/python/sedonadb/src/lib.rs
+++ b/python/sedonadb/src/lib.rs
@@ -134,6 +134,7 @@ fn _lib(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(expr::expr_binary, m)?)?;
m.add_function(wrap_pyfunction!(expr::expr_not, m)?)?;
m.add_function(wrap_pyfunction!(expr::expr_sort_expr, m)?)?;
+ m.add_function(wrap_pyfunction!(schema::raster_type, m)?)?;
m.add_class::<context::InternalContext>()?;
m.add_class::<dataframe::InternalDataFrame>()?;
diff --git a/python/sedonadb/src/schema.rs b/python/sedonadb/src/schema.rs
index 9b02cc612..32466d175 100644
--- a/python/sedonadb/src/schema.rs
+++ b/python/sedonadb/src/schema.rs
@@ -26,6 +26,13 @@ use sedona_schema::schema::SedonaSchema;
use crate::error::PySedonaError;
+#[pyfunction]
+pub fn raster_type() -> PySedonaType {
+ PySedonaType {
+ inner: SedonaType::Raster,
+ }
+}
+
#[pyclass]
pub struct PySedonaSchema {
pub inner: Schema,
diff --git a/python/sedonadb/tests/test_raster.py b/python/sedonadb/tests/test_raster.py
index b714f894e..6ea654249 100644
--- a/python/sedonadb/tests/test_raster.py
+++ b/python/sedonadb/tests/test_raster.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+import pytest
+
from sedonadb.raster import Raster, RasterArray, RasterScalar, RasterType
@@ -67,3 +69,50 @@ def test_raster_to_lit(con):
).to_pandas()
assert t2.iloc[0, 0] == r.width
assert t2.iloc[0, 1] == r.height
+
+
+def test_raster_lazy():
+ # Basic lazy raster creation
+ r = Raster.lazy(
+ uri="s3://bucket/path/to/data.zarr",
+ shape=(512, 1024),
+ dtype="float32",
+ )
+
+ assert r.width == 1024
+ assert r.height == 512
+ assert len(r.bands) == 1
+
+ b = r.bands[0]
+ assert b.source_shape == (512, 1024)
+ assert b.data_type == "float32"
+ assert b.outdb_uri == "s3://bucket/path/to/data.zarr"
+
+ # Lazy raster should have empty data buffer
+ assert len(b.source_data) == 0
+
+ # Accessing data should raise an error for lazy rasters
+ with pytest.raises(ValueError, match="external data"):
+ _ = b.data
+
+
+def test_raster_lazy_with_crs():
+ r = Raster.lazy(
+ uri="s3://bucket/path/to/data.zarr",
+ shape=(256, 256),
+ dtype="uint8",
+ format="zarr",
+ crs="EPSG:4326",
+ )
+
+ assert r.width == 256
+ assert r.height == 256
+ assert r.crs.to_json_dict()["id"] == {"authority": "EPSG", "code": 4326}
+
+
+def test_raster_lazy_invalid_shape():
+ with pytest.raises(ValueError, match="exactly two dimensions"):
+ Raster.lazy(uri="s3://bucket/data.zarr", shape=(10,), dtype="UInt8")
+
+ with pytest.raises(ValueError, match="exactly two dimensions"):
+ Raster.lazy(uri="s3://bucket/data.zarr", shape=(10, 20, 30), dtype="UInt8")