You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@sedona.apache.org by "james-willis (via GitHub)" <gi...@apache.org> on 2026/06/23 20:41:07 UTC
[PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
james-willis opened a new pull request, #1000:
URL: https://github.com/apache/sedona-db/pull/1000
Migrates RS_Clip out of the GDAL raster draft (#704) into its own module, with N-D plane-broadcast support, SQL reference docs, and a benchmark.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589607167
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
Review Comment:
avoided casting by just writing as bytes. no more casting in this flow
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589971316
##########
python/sedonadb/tests/functions/test_rs_clip.py:
##########
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""RS_Clip cross-checked against a rasterio reference implementation.
+
+Every parity test defines the raster once (numpy array + GDAL geotransform,
+written to a CRS-less GeoTIFF), clips it through both engines, and compares
+pixels, geotransform, and nodata exactly. Option permutations run as rows of
+one query so the kernel executes its real array path rather than
+constant-folding literals. Cases rasterio cannot express (empty-mask
+lenient/strict semantics, out-of-range nodata) are asserted directly against
+RS_Clip's documented behavior.
+"""
+
+import numpy as np
+import pytest
+
+from raster_parity import (
+ dtype_min,
+ raster_to_numpy,
+ rasterio_clip,
+ run_clip_rows,
+ write_geotiff,
+)
+
+pytest.importorskip("rasterio")
+pytest.importorskip("shapely")
+
+
+# GDAL-order geotransform: origin (100, 500), 2-wide by 3-tall north-up pixels.
+# With a 7x6 raster the extent is x in [100, 114], y in [482, 500]; pixel
+# boundaries sit at even x and at y = 482 + 3k, so geometry coordinates below
+# deliberately avoid those values (no floor/ceil ambiguity in either engine).
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 3, 6, 7
+
+GEOM_RECT = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8))"
+)
+GEOM_TRIANGLE = "POLYGON ((101.3 498.6, 112.4 496.9, 104.2 483.7, 101.3 498.6))"
+GEOM_HOLE = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8), "
+ "(104.7 492.2, 107.1 492.2, 107.1 489.4, 104.7 489.4, 104.7 492.2))"
+)
+# Extends past the west and north edges of the raster: the crop window is the
+# envelope intersected with the raster extent.
+GEOM_OVERHANG = "POLYGON ((95 505, 106.3 505, 106.3 490.7, 95 490.7, 95 505))"
+# A strip crossing the x = 104 pixel boundary but containing no pixel center
+# (centers sit at odd x): selects nothing unless all_touched.
+GEOM_SLIVER = "POLYGON ((103.6 499, 104.4 499, 104.4 483, 103.6 483, 103.6 499))"
+GEOM_DISJOINT = "POLYGON ((900 900, 910 900, 910 890, 900 890, 900 900))"
+
+# Explicit no_data_value argument and the band nodata baked into the fixture
+# GeoTIFF, per dtype. Values are chosen representable in the dtype and distinct
+# from each other so a mixup shows up in the comparison.
+EXPLICIT_NODATA = {
+ "uint8": 250.0,
+ "uint16": 65000.0,
+ "int16": -9999.0,
+ "int32": -99999.0,
+ "float32": -9999.5,
+ "float64": -12345.5,
+}
+BAND_NODATA = {
+ "uint8": 200.0,
+ "uint16": 60000.0,
+ "int16": -8888.0,
+ "int32": -88888.0,
+ "float32": -8888.5,
+ "float64": -23456.5,
+}
+
+
+def _random_data(dtype, band_nodata=None):
+ """Random (BANDS, HEIGHT, WIDTH) pixels with adversarial plants.
Review Comment:
done
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
Review Comment:
created the raster_testing and RasterEngine components to hold the rasterio and sedonad db comparisons. will extend later this week with a harness for more engines to compare behavior.
##########
python/sedonadb/tests/functions/test_rs_clip.py:
##########
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""RS_Clip cross-checked against a rasterio reference implementation.
+
+Every parity test defines the raster once (numpy array + GDAL geotransform,
+written to a CRS-less GeoTIFF), clips it through both engines, and compares
+pixels, geotransform, and nodata exactly. Option permutations run as rows of
+one query so the kernel executes its real array path rather than
+constant-folding literals. Cases rasterio cannot express (empty-mask
+lenient/strict semantics, out-of-range nodata) are asserted directly against
+RS_Clip's documented behavior.
+"""
+
+import numpy as np
+import pytest
+
+from raster_parity import (
+ dtype_min,
+ raster_to_numpy,
+ rasterio_clip,
+ run_clip_rows,
+ write_geotiff,
+)
+
+pytest.importorskip("rasterio")
+pytest.importorskip("shapely")
+
+
+# GDAL-order geotransform: origin (100, 500), 2-wide by 3-tall north-up pixels.
+# With a 7x6 raster the extent is x in [100, 114], y in [482, 500]; pixel
+# boundaries sit at even x and at y = 482 + 3k, so geometry coordinates below
+# deliberately avoid those values (no floor/ceil ambiguity in either engine).
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 3, 6, 7
+
+GEOM_RECT = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8))"
+)
+GEOM_TRIANGLE = "POLYGON ((101.3 498.6, 112.4 496.9, 104.2 483.7, 101.3 498.6))"
+GEOM_HOLE = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8), "
+ "(104.7 492.2, 107.1 492.2, 107.1 489.4, 104.7 489.4, 104.7 492.2))"
+)
+# Extends past the west and north edges of the raster: the crop window is the
+# envelope intersected with the raster extent.
+GEOM_OVERHANG = "POLYGON ((95 505, 106.3 505, 106.3 490.7, 95 490.7, 95 505))"
+# A strip crossing the x = 104 pixel boundary but containing no pixel center
+# (centers sit at odd x): selects nothing unless all_touched.
+GEOM_SLIVER = "POLYGON ((103.6 499, 104.4 499, 104.4 483, 103.6 483, 103.6 499))"
+GEOM_DISJOINT = "POLYGON ((900 900, 910 900, 910 890, 900 890, 900 900))"
+
+# Explicit no_data_value argument and the band nodata baked into the fixture
+# GeoTIFF, per dtype. Values are chosen representable in the dtype and distinct
+# from each other so a mixup shows up in the comparison.
+EXPLICIT_NODATA = {
+ "uint8": 250.0,
+ "uint16": 65000.0,
+ "int16": -9999.0,
+ "int32": -99999.0,
+ "float32": -9999.5,
+ "float64": -12345.5,
+}
+BAND_NODATA = {
+ "uint8": 200.0,
+ "uint16": 60000.0,
+ "int16": -8888.0,
+ "int32": -88888.0,
+ "float32": -8888.5,
+ "float64": -23456.5,
+}
+
+
+def _random_data(dtype, band_nodata=None):
+ """Random (BANDS, HEIGHT, WIDTH) pixels with adversarial plants.
+
+ The dtype extremes go in opposite corners (outside GEOM_TRIANGLE, so they
+ are overwritten by fill) and the minimum also at (3, 2), whose pixel center
+ (105, 489.5) is inside GEOM_TRIANGLE at either all_touched setting — so an
+ extreme value must also survive a clip verbatim. When ``band_nodata`` is
+ given it is planted at (2, 3), also inside GEOM_TRIANGLE: a source pixel
+ *valued* at the band nodata stays verbatim under a different explicit
+ no_data_value (RS_Clip does not remap it the way rasterio.mask.mask would).
+ """
+ rng = np.random.default_rng(42)
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ data = ((rng.random((BANDS, HEIGHT, WIDTH)) - 0.5) * 200.0).astype(dtype)
+ info = np.finfo(dtype)
+ else:
+ info = np.iinfo(dtype)
+ data = rng.integers(
+ info.min, info.max, size=(BANDS, HEIGHT, WIDTH), dtype=dtype, endpoint=True
+ )
+ data[:, 0, 0] = info.max
+ data[:, -1, -1] = info.min
+ data[:, 3, 2] = info.min
+ if band_nodata is not None:
+ data[:, 2, 3] = band_nodata
+ return data
+
+
+def _assert_clip_matches_rasterio(got, tiff, row, fill):
+ """Compare one RS_Clip result against the rasterio reference clip.
+
+ ``fill`` is the resolved nodata (explicit argument or band nodata or dtype
+ minimum); it is always passed to rasterio explicitly because rasterio's own
+ fallback for a nodata-less band is 0 while RS_Clip's is the dtype minimum.
+ """
+ expected, expected_transform = rasterio_clip(
+ tiff,
+ row["wkt"],
+ all_touched=row["all_touched"],
+ nodata=fill,
+ crop=row["crop"],
+ )
+ if row["band"] != 0:
+ expected = expected[row["band"] - 1 : row["band"]]
+
+ got_pixels, got_transform, got_nodata = raster_to_numpy(got)
+ np.testing.assert_array_equal(got_pixels, expected, err_msg=f"row: {row}")
+ assert got_transform == pytest.approx(expected_transform, rel=1e-12, abs=1e-12)
+ assert got_nodata == [fill] * expected.shape[0], f"row: {row}"
+
+
+@pytest.mark.parametrize("dtype", list(EXPLICIT_NODATA))
+def test_rs_clip_matches_rasterio_permutations(con, tmp_path, dtype):
+ """Cross-product of the optional arguments, one row per combination: band
+ (0 = all bands, 1..3), all_touched, explicit vs band nodata, crop — over
+ two geometries chosen so every axis discriminates. GEOM_TRIANGLE's
+ diagonal edges make all_touched matter but its envelope spans the whole
+ raster (crop is a no-op there); GEOM_HOLE's interior window has nonzero
+ offsets, so its crop=True rows shift the transform and shrink the shape
+ while crop=False rows must fill outside an interior window."""
+ tiff = tmp_path / f"clip_{dtype}.tif"
+ write_geotiff(
+ tiff,
+ _random_data(dtype, band_nodata=BAND_NODATA[dtype]),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA[dtype],
+ )
+
+ rows = [
+ {
+ "wkt": wkt,
+ "band": band,
+ "all_touched": at,
+ "nodata": nd,
+ "crop": crop,
+ }
+ for wkt in (GEOM_TRIANGLE, GEOM_HOLE)
+ for band in (0, 1, 2, 3)
+ for at in (False, True)
+ for nd in (None, EXPLICIT_NODATA[dtype])
+ for crop in (True, False)
+ ]
+ results = run_clip_rows(con, tiff, rows)
+
+ for row, got in zip(rows, results):
+ assert got is not None, f"row: {row}"
+ fill = row["nodata"] if row["nodata"] is not None else BAND_NODATA[dtype]
+ _assert_clip_matches_rasterio(got, tiff, row, fill)
+
+
+@pytest.mark.parametrize("all_touched", [False, True])
+@pytest.mark.parametrize(
+ "wkt",
+ [GEOM_RECT, GEOM_TRIANGLE, GEOM_HOLE, GEOM_OVERHANG],
+ ids=["rect", "triangle", "hole", "overhang"],
+)
+def test_rs_clip_matches_rasterio_geometries(con, tmp_path, wkt, all_touched):
+ """Geometry shapes: axis-aligned, diagonal edges, interior ring, and an
+ envelope overhanging the raster edge (crop window = envelope ∩ extent)."""
+ tiff = tmp_path / "clip_geom.tif"
+ write_geotiff(
+ tiff,
+ _random_data("uint8"),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA["uint8"],
+ )
+
+ row = {
+ "wkt": wkt,
+ "band": 0,
+ "all_touched": all_touched,
+ "nodata": None,
+ "crop": True,
+ }
+ (got,) = run_clip_rows(con, tiff, [row])
+ assert got is not None
+ _assert_clip_matches_rasterio(got, tiff, row, BAND_NODATA["uint8"])
+
+
+@pytest.mark.parametrize("dtype", list(EXPLICIT_NODATA) + ["int8", "int64", "uint64"])
+def test_rs_clip_default_nodata_sentinel_matches_rasterio(con, tmp_path, dtype):
+ """No explicit nodata and no band nodata: masked pixels get the dtype
+ minimum (exact even for 64-bit integers — no f64 round-trip), and the
+ output band records it. GEOM_TRIANGLE leaves most of its crop window
+ outside the geometry, so the sentinel is actually written to pixels, not
+ just metadata. rasterio is handed the same sentinel because its own
+ nodata-less fallback is 0."""
+ tiff = tmp_path / f"clip_sentinel_{dtype}.tif"
+ write_geotiff(tiff, _random_data(dtype), gdal_transform=GDAL_TRANSFORM)
+
+ row = {
+ "wkt": GEOM_TRIANGLE,
+ "band": 1,
+ "all_touched": False,
+ "nodata": None,
+ "crop": True,
+ }
+ (got,) = run_clip_rows(con, tiff, [row])
+ assert got is not None
+ _assert_clip_matches_rasterio(got, tiff, row, dtype_min(dtype))
+
+
+def test_rs_clip_signature_defaults(con, tmp_path):
+ """Each shorter signature behaves as the full 7-arg form with the defaults
+ filled in: all_touched = false, no_data_value = the band's own, crop = true,
+ lenient = true."""
+ tiff = tmp_path / "clip_sigs.tif"
+ write_geotiff(
+ tiff,
+ _random_data("uint8"),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA["uint8"],
+ )
+
+ def clip(args):
+ tab = con.sql(
+ f"SELECT RS_Clip(RS_FromPath($1), 1, ST_GeomFromText($2){args}) AS r",
+ params=(str(tiff), GEOM_TRIANGLE),
+ ).to_arrow_table()
+ return raster_to_numpy(tab["r"][0].as_py())
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#issuecomment-4898048594
I think it would be rather difficult to have enough raster tiles that further optimizing the geometry iteration would make a difference given that this is reaching all the way into GDAL and pulling pixels.
For what it's worth, I believe the trivial version of this (north up) can be zero copy using views. (Whether that is the best choice for any particular chain of operations is a different story, so maybe not now).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589959514
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "prantogg (via GitHub)" <gi...@apache.org>.
prantogg commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3546481011
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height).
+ /// `None` means the full original raster extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ // Create a mask raster (same dimensions as input)
+ let mask_dataset = MemDatasetBuilder::create(gdal, width, height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+
+ // Set the same geotransform as the input raster
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+ mask_dataset
+ .set_geo_transform(&geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>((0, 0), (width, height), (width, height), None)
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // Check if there are any non-zero pixels in the mask (i.e. geometry intersects raster)
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ // Compute crop window if crop=true
+ let crop_window = if crop {
+ compute_crop_window(mask, width, height)
+ } else {
+ None
+ };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(plane_bytes, mask, width, height, &data_type, nodata)?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the minimal bounding pixel window that contains all non-zero mask pixels.
+fn compute_crop_window(mask: &[u8], width: usize, height: usize) -> Option<CropWindow> {
+ let mut min_col = width;
+ let mut max_col = 0usize;
+ let mut min_row = height;
+ let mut max_row = 0usize;
+
+ for row in 0..height {
+ for col in 0..width {
+ if mask[row * width + col] != 0 {
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+ }
+ }
+
+ if min_col > max_col || min_row > max_row {
+ return None; // no non-zero pixels (shouldn't happen if caller checked)
+ }
+
+ Some(CropWindow {
+ col_off: min_col,
+ row_off: min_row,
+ width: max_col - min_col + 1,
+ height: max_row - min_row + 1,
+ })
+}
Review Comment:
The crop window here is the tight bbox of the selected mask pixels, but the prior art crops to the geometry envelope ∩ raster extent, snapped to the grid — PostGIS `ST_Clip` (which the module doc cites), `gdalwarp -crop_to_cutline`, and Sedona Spark's `RS_Clip` all do the latter, keeping unselected envelope pixels as nodata padding.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589643929
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
Review Comment:
Won't datafusion cast the input argument for the no data value to a f64 before we ever get to see the original value? so were kind of screwed here in terms of e.g. representing 2^60 faithfully.
Anyway, the shared utilities now live in `sedona_raster::traits` and are used by `rs_set_band_nodata` and `rs_value` as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3583191931
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
Review Comment:
Should this be a method on `Band`?
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
+ raw = band._py_field("nodata")
+ if raw is None:
+ return None
+ type_id = BAND_DATA_TYPE_IDS[band.data_type]
+ return struct.unpack("<" + BAND_DATA_TYPE_STRUCT_CHARS[type_id], raw)[0]
+
+
+def raster_to_numpy(raster: Raster):
+ """Extract ``(array(bands, h, w), gdal_transform, [nodata per band])``."""
+ arrays = [band.to_numpy() for band in raster.bands]
+ nodatas = [band_nodata(band) for band in raster.bands]
+ return np.stack(arrays), tuple(raster.transform), nodatas
+
+
+def run_clip_rows(con, tiff_path, rows):
+ """Run ``RS_Clip`` once over an N-row table, one row per parameter combo.
+
+ Options arrive as table columns rather than literals so the kernel runs
+ its real array execution path (literals constant-fold). Each row is a dict
+ with keys ``wkt``, ``band``, ``all_touched``, ``nodata`` (None = use the
+ band's own), and ``crop``. Returns a list of ``Raster`` (or None for NULL
+ rows) in row order.
+ """
+ table = pa.table(
+ {
+ "idx": pa.array(range(len(rows)), type=pa.int64()),
+ "path": pa.array([str(tiff_path)] * len(rows), type=pa.utf8()),
+ "wkt": pa.array([r["wkt"] for r in rows], type=pa.utf8()),
+ "band": pa.array([r["band"] for r in rows], type=pa.int32()),
+ "all_touched": pa.array([r["all_touched"] for r in rows], type=pa.bool_()),
+ "nodata": pa.array([r["nodata"] for r in rows], type=pa.float64()),
+ "crop": pa.array([r["crop"] for r in rows], type=pa.bool_()),
+ }
+ )
+ view = "raster_parity_clip_rows"
+ con.create_data_frame(table).to_view(view, overwrite=True)
+ try:
+ result = con.sql(
+ "SELECT RS_Clip(RS_FromPath(path), band, ST_GeomFromText(wkt),"
+ f" all_touched, nodata, crop) AS r FROM {view} ORDER BY idx"
+ ).to_arrow_table()["r"]
Review Comment:
Does it make more sense to just use the python api instead of an f-string?
```python
t = con.create_data_frame(table)
t.select(
r=t.path.funcs.rs_from_path().funcs.rs_clip(
t.band,
con.funcs.st_geom_from_text(t.wkt),
t.all_touched,
t.nodata,
t.crop
)
)
```
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
+ raw = band._py_field("nodata")
+ if raw is None:
+ return None
+ type_id = BAND_DATA_TYPE_IDS[band.data_type]
+ return struct.unpack("<" + BAND_DATA_TYPE_STRUCT_CHARS[type_id], raw)[0]
+
+
+def raster_to_numpy(raster: Raster):
+ """Extract ``(array(bands, h, w), gdal_transform, [nodata per band])``."""
Review Comment:
Should this be a method on `Raster`? I suppose the stacking would wipe out zero copy but that could be documented.
##########
python/sedonadb/tests/functions/test_rs_clip.py:
##########
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""RS_Clip cross-checked against a rasterio reference implementation.
+
+Every parity test defines the raster once (numpy array + GDAL geotransform,
+written to a CRS-less GeoTIFF), clips it through both engines, and compares
+pixels, geotransform, and nodata exactly. Option permutations run as rows of
+one query so the kernel executes its real array path rather than
+constant-folding literals. Cases rasterio cannot express (empty-mask
+lenient/strict semantics, out-of-range nodata) are asserted directly against
+RS_Clip's documented behavior.
+"""
+
+import numpy as np
+import pytest
+
+from raster_parity import (
+ dtype_min,
+ raster_to_numpy,
+ rasterio_clip,
+ run_clip_rows,
+ write_geotiff,
+)
+
+pytest.importorskip("rasterio")
+pytest.importorskip("shapely")
+
+
+# GDAL-order geotransform: origin (100, 500), 2-wide by 3-tall north-up pixels.
+# With a 7x6 raster the extent is x in [100, 114], y in [482, 500]; pixel
+# boundaries sit at even x and at y = 482 + 3k, so geometry coordinates below
+# deliberately avoid those values (no floor/ceil ambiguity in either engine).
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 3, 6, 7
+
+GEOM_RECT = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8))"
+)
+GEOM_TRIANGLE = "POLYGON ((101.3 498.6, 112.4 496.9, 104.2 483.7, 101.3 498.6))"
+GEOM_HOLE = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8), "
+ "(104.7 492.2, 107.1 492.2, 107.1 489.4, 104.7 489.4, 104.7 492.2))"
+)
+# Extends past the west and north edges of the raster: the crop window is the
+# envelope intersected with the raster extent.
+GEOM_OVERHANG = "POLYGON ((95 505, 106.3 505, 106.3 490.7, 95 490.7, 95 505))"
+# A strip crossing the x = 104 pixel boundary but containing no pixel center
+# (centers sit at odd x): selects nothing unless all_touched.
+GEOM_SLIVER = "POLYGON ((103.6 499, 104.4 499, 104.4 483, 103.6 483, 103.6 499))"
+GEOM_DISJOINT = "POLYGON ((900 900, 910 900, 910 890, 900 890, 900 900))"
+
+# Explicit no_data_value argument and the band nodata baked into the fixture
+# GeoTIFF, per dtype. Values are chosen representable in the dtype and distinct
+# from each other so a mixup shows up in the comparison.
+EXPLICIT_NODATA = {
+ "uint8": 250.0,
+ "uint16": 65000.0,
+ "int16": -9999.0,
+ "int32": -99999.0,
+ "float32": -9999.5,
+ "float64": -12345.5,
+}
+BAND_NODATA = {
+ "uint8": 200.0,
+ "uint16": 60000.0,
+ "int16": -8888.0,
+ "int32": -88888.0,
+ "float32": -8888.5,
+ "float64": -23456.5,
+}
+
+
+def _random_data(dtype, band_nodata=None):
+ """Random (BANDS, HEIGHT, WIDTH) pixels with adversarial plants.
+
+ The dtype extremes go in opposite corners (outside GEOM_TRIANGLE, so they
+ are overwritten by fill) and the minimum also at (3, 2), whose pixel center
+ (105, 489.5) is inside GEOM_TRIANGLE at either all_touched setting — so an
+ extreme value must also survive a clip verbatim. When ``band_nodata`` is
+ given it is planted at (2, 3), also inside GEOM_TRIANGLE: a source pixel
+ *valued* at the band nodata stays verbatim under a different explicit
+ no_data_value (RS_Clip does not remap it the way rasterio.mask.mask would).
+ """
+ rng = np.random.default_rng(42)
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ data = ((rng.random((BANDS, HEIGHT, WIDTH)) - 0.5) * 200.0).astype(dtype)
+ info = np.finfo(dtype)
+ else:
+ info = np.iinfo(dtype)
+ data = rng.integers(
+ info.min, info.max, size=(BANDS, HEIGHT, WIDTH), dtype=dtype, endpoint=True
+ )
+ data[:, 0, 0] = info.max
+ data[:, -1, -1] = info.min
+ data[:, 3, 2] = info.min
+ if band_nodata is not None:
+ data[:, 2, 3] = band_nodata
+ return data
+
+
+def _assert_clip_matches_rasterio(got, tiff, row, fill):
+ """Compare one RS_Clip result against the rasterio reference clip.
+
+ ``fill`` is the resolved nodata (explicit argument or band nodata or dtype
+ minimum); it is always passed to rasterio explicitly because rasterio's own
+ fallback for a nodata-less band is 0 while RS_Clip's is the dtype minimum.
+ """
+ expected, expected_transform = rasterio_clip(
+ tiff,
+ row["wkt"],
+ all_touched=row["all_touched"],
+ nodata=fill,
+ crop=row["crop"],
+ )
+ if row["band"] != 0:
+ expected = expected[row["band"] - 1 : row["band"]]
+
+ got_pixels, got_transform, got_nodata = raster_to_numpy(got)
+ np.testing.assert_array_equal(got_pixels, expected, err_msg=f"row: {row}")
+ assert got_transform == pytest.approx(expected_transform, rel=1e-12, abs=1e-12)
+ assert got_nodata == [fill] * expected.shape[0], f"row: {row}"
+
+
+@pytest.mark.parametrize("dtype", list(EXPLICIT_NODATA))
+def test_rs_clip_matches_rasterio_permutations(con, tmp_path, dtype):
+ """Cross-product of the optional arguments, one row per combination: band
+ (0 = all bands, 1..3), all_touched, explicit vs band nodata, crop — over
+ two geometries chosen so every axis discriminates. GEOM_TRIANGLE's
+ diagonal edges make all_touched matter but its envelope spans the whole
+ raster (crop is a no-op there); GEOM_HOLE's interior window has nonzero
+ offsets, so its crop=True rows shift the transform and shrink the shape
+ while crop=False rows must fill outside an interior window."""
+ tiff = tmp_path / f"clip_{dtype}.tif"
+ write_geotiff(
+ tiff,
+ _random_data(dtype, band_nodata=BAND_NODATA[dtype]),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA[dtype],
+ )
+
+ rows = [
+ {
+ "wkt": wkt,
+ "band": band,
+ "all_touched": at,
+ "nodata": nd,
+ "crop": crop,
+ }
+ for wkt in (GEOM_TRIANGLE, GEOM_HOLE)
+ for band in (0, 1, 2, 3)
+ for at in (False, True)
+ for nd in (None, EXPLICIT_NODATA[dtype])
+ for crop in (True, False)
+ ]
+ results = run_clip_rows(con, tiff, rows)
+
+ for row, got in zip(rows, results):
+ assert got is not None, f"row: {row}"
+ fill = row["nodata"] if row["nodata"] is not None else BAND_NODATA[dtype]
+ _assert_clip_matches_rasterio(got, tiff, row, fill)
+
+
+@pytest.mark.parametrize("all_touched", [False, True])
+@pytest.mark.parametrize(
+ "wkt",
+ [GEOM_RECT, GEOM_TRIANGLE, GEOM_HOLE, GEOM_OVERHANG],
+ ids=["rect", "triangle", "hole", "overhang"],
+)
+def test_rs_clip_matches_rasterio_geometries(con, tmp_path, wkt, all_touched):
+ """Geometry shapes: axis-aligned, diagonal edges, interior ring, and an
+ envelope overhanging the raster edge (crop window = envelope ∩ extent)."""
+ tiff = tmp_path / "clip_geom.tif"
+ write_geotiff(
+ tiff,
+ _random_data("uint8"),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA["uint8"],
+ )
+
+ row = {
+ "wkt": wkt,
+ "band": 0,
+ "all_touched": all_touched,
+ "nodata": None,
+ "crop": True,
+ }
+ (got,) = run_clip_rows(con, tiff, [row])
+ assert got is not None
+ _assert_clip_matches_rasterio(got, tiff, row, BAND_NODATA["uint8"])
+
+
+@pytest.mark.parametrize("dtype", list(EXPLICIT_NODATA) + ["int8", "int64", "uint64"])
+def test_rs_clip_default_nodata_sentinel_matches_rasterio(con, tmp_path, dtype):
+ """No explicit nodata and no band nodata: masked pixels get the dtype
+ minimum (exact even for 64-bit integers — no f64 round-trip), and the
+ output band records it. GEOM_TRIANGLE leaves most of its crop window
+ outside the geometry, so the sentinel is actually written to pixels, not
+ just metadata. rasterio is handed the same sentinel because its own
+ nodata-less fallback is 0."""
+ tiff = tmp_path / f"clip_sentinel_{dtype}.tif"
+ write_geotiff(tiff, _random_data(dtype), gdal_transform=GDAL_TRANSFORM)
+
+ row = {
+ "wkt": GEOM_TRIANGLE,
+ "band": 1,
+ "all_touched": False,
+ "nodata": None,
+ "crop": True,
+ }
+ (got,) = run_clip_rows(con, tiff, [row])
+ assert got is not None
+ _assert_clip_matches_rasterio(got, tiff, row, dtype_min(dtype))
+
+
+def test_rs_clip_signature_defaults(con, tmp_path):
+ """Each shorter signature behaves as the full 7-arg form with the defaults
+ filled in: all_touched = false, no_data_value = the band's own, crop = true,
+ lenient = true."""
+ tiff = tmp_path / "clip_sigs.tif"
+ write_geotiff(
+ tiff,
+ _random_data("uint8"),
+ gdal_transform=GDAL_TRANSFORM,
+ nodata=BAND_NODATA["uint8"],
+ )
+
+ def clip(args):
+ tab = con.sql(
+ f"SELECT RS_Clip(RS_FromPath($1), 1, ST_GeomFromText($2){args}) AS r",
+ params=(str(tiff), GEOM_TRIANGLE),
+ ).to_arrow_table()
+ return raster_to_numpy(tab["r"][0].as_py())
Review Comment:
The Python API would make this f-string/parameterized SQL/inline `def` combination less hacky feeling. If you use the `expr.rs.clip()` accessor (which should have been automatically generated in sedona-expr), you can even use named params.
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
Review Comment:
Should this go in `testing.py` (or maybe a new `raster_testing.py`)? Maybe `class RasterEngine` with methods `clip()`, etc that could be implemented for SedonaDB and rasterio?
##########
python/sedonadb/tests/functions/test_rs_clip.py:
##########
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""RS_Clip cross-checked against a rasterio reference implementation.
+
+Every parity test defines the raster once (numpy array + GDAL geotransform,
+written to a CRS-less GeoTIFF), clips it through both engines, and compares
+pixels, geotransform, and nodata exactly. Option permutations run as rows of
+one query so the kernel executes its real array path rather than
+constant-folding literals. Cases rasterio cannot express (empty-mask
+lenient/strict semantics, out-of-range nodata) are asserted directly against
+RS_Clip's documented behavior.
+"""
+
+import numpy as np
+import pytest
+
+from raster_parity import (
+ dtype_min,
+ raster_to_numpy,
+ rasterio_clip,
+ run_clip_rows,
+ write_geotiff,
+)
+
+pytest.importorskip("rasterio")
+pytest.importorskip("shapely")
+
+
+# GDAL-order geotransform: origin (100, 500), 2-wide by 3-tall north-up pixels.
+# With a 7x6 raster the extent is x in [100, 114], y in [482, 500]; pixel
+# boundaries sit at even x and at y = 482 + 3k, so geometry coordinates below
+# deliberately avoid those values (no floor/ceil ambiguity in either engine).
+GDAL_TRANSFORM = (100.0, 2.0, 0.0, 500.0, 0.0, -3.0)
+BANDS, HEIGHT, WIDTH = 3, 6, 7
+
+GEOM_RECT = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8))"
+)
+GEOM_TRIANGLE = "POLYGON ((101.3 498.6, 112.4 496.9, 104.2 483.7, 101.3 498.6))"
+GEOM_HOLE = (
+ "POLYGON ((102.6 495.8, 109.3 495.8, 109.3 485.9, 102.6 485.9, 102.6 495.8), "
+ "(104.7 492.2, 107.1 492.2, 107.1 489.4, 104.7 489.4, 104.7 492.2))"
+)
+# Extends past the west and north edges of the raster: the crop window is the
+# envelope intersected with the raster extent.
+GEOM_OVERHANG = "POLYGON ((95 505, 106.3 505, 106.3 490.7, 95 490.7, 95 505))"
+# A strip crossing the x = 104 pixel boundary but containing no pixel center
+# (centers sit at odd x): selects nothing unless all_touched.
+GEOM_SLIVER = "POLYGON ((103.6 499, 104.4 499, 104.4 483, 103.6 483, 103.6 499))"
+GEOM_DISJOINT = "POLYGON ((900 900, 910 900, 910 890, 900 890, 900 900))"
+
+# Explicit no_data_value argument and the band nodata baked into the fixture
+# GeoTIFF, per dtype. Values are chosen representable in the dtype and distinct
+# from each other so a mixup shows up in the comparison.
+EXPLICIT_NODATA = {
+ "uint8": 250.0,
+ "uint16": 65000.0,
+ "int16": -9999.0,
+ "int32": -99999.0,
+ "float32": -9999.5,
+ "float64": -12345.5,
+}
+BAND_NODATA = {
+ "uint8": 200.0,
+ "uint16": 60000.0,
+ "int16": -8888.0,
+ "int32": -88888.0,
+ "float32": -8888.5,
+ "float64": -23456.5,
+}
+
+
+def _random_data(dtype, band_nodata=None):
+ """Random (BANDS, HEIGHT, WIDTH) pixels with adversarial plants.
Review Comment:
This would also be a good addition to `raster_testing.py`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589467819
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589647346
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
+fn band_data_type_max(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8 => u8::MAX as f64,
+ BandDataType::UInt16 => u16::MAX as f64,
+ BandDataType::UInt32 => u32::MAX as f64,
+ BandDataType::UInt64 => u64::MAX as f64,
+ BandDataType::Int8 => i8::MAX as f64,
+ BandDataType::Int16 => i16::MAX as f64,
+ BandDataType::Int32 => i32::MAX as f64,
+ BandDataType::Int64 => i64::MAX as f64,
+ BandDataType::Float32 => f32::MAX as f64,
+ BandDataType::Float64 => f64::MAX,
+ }
+}
+
+fn calc_num_iterations(args: &[ColumnarValue]) -> usize {
+ for arg in args {
+ if let ColumnarValue::Array(array) = arg {
+ return array.len();
+ }
+ }
+ 1
+}
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#issuecomment-4897941764
@paleolimbot should we hoist the scalar geometry case? Is that a common case we want to optimize?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589594950
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#issuecomment-4898248902
> can be zero copy using views
I was thinking this but never finished the views feature
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589957873
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
+ raw = band._py_field("nodata")
+ if raw is None:
+ return None
+ type_id = BAND_DATA_TYPE_IDS[band.data_type]
+ return struct.unpack("<" + BAND_DATA_TYPE_STRUCT_CHARS[type_id], raw)[0]
+
+
+def raster_to_numpy(raster: Raster):
+ """Extract ``(array(bands, h, w), gdal_transform, [nodata per band])``."""
+ arrays = [band.to_numpy() for band in raster.bands]
+ nodatas = [band_nodata(band) for band in raster.bands]
+ return np.stack(arrays), tuple(raster.transform), nodatas
+
+
+def run_clip_rows(con, tiff_path, rows):
+ """Run ``RS_Clip`` once over an N-row table, one row per parameter combo.
+
+ Options arrive as table columns rather than literals so the kernel runs
+ its real array execution path (literals constant-fold). Each row is a dict
+ with keys ``wkt``, ``band``, ``all_touched``, ``nodata`` (None = use the
+ band's own), and ``crop``. Returns a list of ``Raster`` (or None for NULL
+ rows) in row order.
+ """
+ table = pa.table(
+ {
+ "idx": pa.array(range(len(rows)), type=pa.int64()),
+ "path": pa.array([str(tiff_path)] * len(rows), type=pa.utf8()),
+ "wkt": pa.array([r["wkt"] for r in rows], type=pa.utf8()),
+ "band": pa.array([r["band"] for r in rows], type=pa.int32()),
+ "all_touched": pa.array([r["all_touched"] for r in rows], type=pa.bool_()),
+ "nodata": pa.array([r["nodata"] for r in rows], type=pa.float64()),
+ "crop": pa.array([r["crop"] for r in rows], type=pa.bool_()),
+ }
+ )
+ view = "raster_parity_clip_rows"
+ con.create_data_frame(table).to_view(view, overwrite=True)
+ try:
+ result = con.sql(
+ "SELECT RS_Clip(RS_FromPath(path), band, ST_GeomFromText(wkt),"
+ f" all_touched, nodata, crop) AS r FROM {view} ORDER BY idx"
+ ).to_arrow_table()["r"]
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589586489
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height).
+ /// `None` means the full original raster extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ // Create a mask raster (same dimensions as input)
+ let mask_dataset = MemDatasetBuilder::create(gdal, width, height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+
+ // Set the same geotransform as the input raster
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+ mask_dataset
+ .set_geo_transform(&geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>((0, 0), (width, height), (width, height), None)
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // Check if there are any non-zero pixels in the mask (i.e. geometry intersects raster)
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ // Compute crop window if crop=true
+ let crop_window = if crop {
+ compute_crop_window(mask, width, height)
+ } else {
+ None
+ };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(plane_bytes, mask, width, height, &data_type, nodata)?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the minimal bounding pixel window that contains all non-zero mask pixels.
+fn compute_crop_window(mask: &[u8], width: usize, height: usize) -> Option<CropWindow> {
+ let mut min_col = width;
+ let mut max_col = 0usize;
+ let mut min_row = height;
+ let mut max_row = 0usize;
+
+ for row in 0..height {
+ for col in 0..width {
+ if mask[row * width + col] != 0 {
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+ }
+ }
+
+ if min_col > max_col || min_row > max_row {
+ return None; // no non-zero pixels (shouldn't happen if caller checked)
+ }
+
+ Some(CropWindow {
+ col_off: min_col,
+ row_off: min_row,
+ width: max_col - min_col + 1,
+ height: max_row - min_row + 1,
+ })
+}
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#issuecomment-4996907508
merging to continue working on the parity harness, happy to take more feedback into a new PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3584131064
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
+fn band_data_type_max(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8 => u8::MAX as f64,
+ BandDataType::UInt16 => u16::MAX as f64,
+ BandDataType::UInt32 => u32::MAX as f64,
+ BandDataType::UInt64 => u64::MAX as f64,
+ BandDataType::Int8 => i8::MAX as f64,
+ BandDataType::Int16 => i16::MAX as f64,
+ BandDataType::Int32 => i32::MAX as f64,
+ BandDataType::Int64 => i64::MAX as f64,
+ BandDataType::Float32 => f32::MAX as f64,
+ BandDataType::Float64 => f64::MAX,
+ }
+}
+
+fn calc_num_iterations(args: &[ColumnarValue]) -> usize {
+ for arg in args {
+ if let ColumnarValue::Array(array) = arg {
+ return array.len();
+ }
+ }
+ 1
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow_array::StructArray;
+ use sedona_expr::scalar_udf::SedonaScalarKernel;
+ use sedona_raster::array::RasterStructArray;
+ use sedona_schema::crs::deserialize_crs;
+ use sedona_schema::datatypes::Edges;
+ use sedona_testing::raster_spec::RasterSpec;
+
+ fn wkb_from_wkt(gdal: &sedona_gdal::gdal::Gdal, wkt: &str) -> Result<Vec<u8>> {
+ let geometry = gdal.geometry_from_wkt(wkt).unwrap();
+ geometry.wkb().map_err(|e| exec_datafusion_err!("{e}"))
+ }
Review Comment:
found it. using it now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#issuecomment-4984281790
Heads up: I've pushed the review follow-ups (Band.nodata / Raster.to_numpy accessors, the sedonadb.raster_testing engine harness, expr-accessor tests) but GitHub is being laggy syncing the branch into the PR — jw/rs-clip is at 171fd09d4 if the Commits/Files tabs look stale.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3575602813
##########
docs/reference/sql/rs_clip.qmd:
##########
@@ -0,0 +1,131 @@
+---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+title: RS_Clip
+description: >
+ Clips a raster to a geometry, setting pixels outside the geometry to nodata
+ and (by default) cropping to the geometry's bounding box.
+kernels:
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - {name: no_data_value, type: double}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - {name: no_data_value, type: double}
+ - {name: crop, type: boolean}
+ - returns: raster
+ args:
+ - raster
+ - name: band
+ type: integer
+ description: >
+ 1-based index of the band to clip, or 0 to clip every band. A NULL
+ band yields a NULL result.
+ - name: geom
+ type: geometry
+ description: >
+ Clip geometry. Reprojected into the raster's CRS when both carry one;
+ it is an error for exactly one side to have a CRS.
+ - name: all_touched
+ type: boolean
+ description: >
+ If true, keep every pixel the geometry touches; otherwise keep only
+ pixels whose center falls inside it. Defaults to false.
+ - name: no_data_value
+ type: double
+ description: >
+ Value written to pixels outside the geometry. Defaults to the band's
+ own nodata value, or the band data type's minimum if it has none. It is
+ an error if the value does not fit the band's data type.
+ - name: crop
+ type: boolean
+ description: Crop the output to the geometry's bounding box. Defaults to true.
+ - name: lenient
+ type: boolean
+ description: >
+ If true, a geometry that does not intersect the raster yields NULL;
+ if false, it raises an error. Defaults to true.
+---
+
+## Description
+
+`RS_Clip` clips a raster to a geometry, in the spirit of PostGIS `ST_Clip`.
Review Comment:
Can you mark this as experimental so we can update the slew of positional arguments in a future update?
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
Review Comment:
You can use the engine from the `SedonaOptions` now that it's available!
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
Review Comment:
I am not sure that int64::MIN is non-lossy when it goes through f64 (which would mean an unrecognized nondata value). You could template this function on output type (or go the GDAL route, where there's often dedicated functions for i64/u64
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
+fn band_data_type_max(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8 => u8::MAX as f64,
+ BandDataType::UInt16 => u16::MAX as f64,
+ BandDataType::UInt32 => u32::MAX as f64,
+ BandDataType::UInt64 => u64::MAX as f64,
+ BandDataType::Int8 => i8::MAX as f64,
+ BandDataType::Int16 => i16::MAX as f64,
+ BandDataType::Int32 => i32::MAX as f64,
+ BandDataType::Int64 => i64::MAX as f64,
+ BandDataType::Float32 => f32::MAX as f64,
+ BandDataType::Float64 => f64::MAX,
+ }
+}
+
+fn calc_num_iterations(args: &[ColumnarValue]) -> usize {
+ for arg in args {
+ if let ColumnarValue::Array(array) = arg {
+ return array.len();
+ }
+ }
+ 1
+}
Review Comment:
This is probably something that needs to be fixed with the executor rather than duplicated here
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
Review Comment:
Can these be associated functions on BandaDataType to reduce the chance the will get duplicated?
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
Review Comment:
Should you just put all the args into the exec and make the exec flexible enough to handle the arg order here?
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
Review Comment:
This should also do a better job with i64/u64 nodata values (or error for these cases).
In general the nodata handling of i64 and u64 has come up a lot and will probably come up in every function that takes a nodata value argument...I am not sure where the utilities should go, but there should be some so that we don't reinvent/make this mistake every time.
The "right" way to handle the nodata value is to cast the nodata_value argument to the band's data type (rather than f64). These functions can use associated types to be generic on the output type.
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1491 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ nodata,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata: f64,
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_bytes = nodata_f64_to_bytes(nodata, data_type);
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(&nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
+
+ // Spatial extent after the clip. `spatial_dims`/`spatial_shape` are kept in
+ // the raster's own axis order (X-first, as the readers emit), so map each
+ // spatial dim to its clipped size by name rather than assuming an order.
+ let spatial_dims = original_raster.spatial_dims();
+ let spatial_shape: Vec<i64> = match clipped_data.crop_window {
+ None => original_raster.spatial_shape().to_vec(),
+ Some(cw) => {
+ let x_dim = original_raster.x_dim();
+ spatial_dims
+ .iter()
+ .map(|&d| {
+ if d == x_dim {
+ cw.width as i64
+ } else {
+ cw.height as i64
+ }
+ })
+ .collect()
+ }
+ };
+
+ builder
+ .start_raster_nd(
+ &transform,
+ &spatial_dims,
+ &spatial_shape,
+ original_raster.crs(),
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start raster: {}", e))?;
+
+ for band in &clipped_data.bands {
+ let dim_names: Vec<&str> = band.dim_names.iter().map(String::as_str).collect();
+ builder
+ .start_band_nd(
+ band.name.as_deref(),
+ &dim_names,
+ &band.shape,
+ band.data_type,
+ Some(&band.nodata),
+ None,
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to start band: {}", e))?;
+ builder.band_data_writer().append_value(&band.data);
+ builder
+ .finish_band()
+ .map_err(|e| exec_datafusion_err!("Failed to finish band: {}", e))?;
+ }
+
+ builder
+ .finish_raster()
+ .map_err(|e| exec_datafusion_err!("Failed to finish raster: {}", e))?;
+
+ Ok(())
+}
+
+/// The minimum representable value of a band data type, as `f64` — the default
+/// nodata sentinel when neither an explicit value nor the band's own nodata is
+/// available, so masked-out pixels stay distinguishable from real data.
+fn band_data_type_min(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8
+ | BandDataType::UInt16
+ | BandDataType::UInt32
+ | BandDataType::UInt64 => 0.0,
+ BandDataType::Int8 => i8::MIN as f64,
+ BandDataType::Int16 => i16::MIN as f64,
+ BandDataType::Int32 => i32::MIN as f64,
+ BandDataType::Int64 => i64::MIN as f64,
+ BandDataType::Float32 => f32::MIN as f64,
+ BandDataType::Float64 => f64::MIN,
+ }
+}
+
+/// The maximum representable value of a band data type, as `f64` — paired with
+/// [`band_data_type_min`] to range-check a custom `no_data_value` before it is
+/// narrowed to the band's type (which would otherwise saturate silently).
+fn band_data_type_max(data_type: &BandDataType) -> f64 {
+ match data_type {
+ BandDataType::UInt8 => u8::MAX as f64,
+ BandDataType::UInt16 => u16::MAX as f64,
+ BandDataType::UInt32 => u32::MAX as f64,
+ BandDataType::UInt64 => u64::MAX as f64,
+ BandDataType::Int8 => i8::MAX as f64,
+ BandDataType::Int16 => i16::MAX as f64,
+ BandDataType::Int32 => i32::MAX as f64,
+ BandDataType::Int64 => i64::MAX as f64,
+ BandDataType::Float32 => f32::MAX as f64,
+ BandDataType::Float64 => f64::MAX,
+ }
+}
+
+fn calc_num_iterations(args: &[ColumnarValue]) -> usize {
+ for arg in args {
+ if let ColumnarValue::Array(array) = arg {
+ return array.len();
+ }
+ }
+ 1
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow_array::StructArray;
+ use sedona_expr::scalar_udf::SedonaScalarKernel;
+ use sedona_raster::array::RasterStructArray;
+ use sedona_schema::crs::deserialize_crs;
+ use sedona_schema::datatypes::Edges;
+ use sedona_testing::raster_spec::RasterSpec;
+
+ fn wkb_from_wkt(gdal: &sedona_gdal::gdal::Gdal, wkt: &str) -> Result<Vec<u8>> {
+ let geometry = gdal.geometry_from_wkt(wkt).unwrap();
+ geometry.wkb().map_err(|e| exec_datafusion_err!("{e}"))
+ }
Review Comment:
We almost certainly have this utility in sedona-testing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3600131092
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1435 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::ArrayRef;
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs, with_crs_engine};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_dataset_provider::configure_thread_local_options;
+use sedona_raster::traits::nodata_f64_to_bytes;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = RasterExecutor::num_iterations_over(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
Review Comment:
We can file a follow-up to fix this cast, which will loose precision for i64 and u64 nodata values and is handled differently in a few PRs. The solution is to resolve the arraydata and add add an accessor for the appropriate slice of the data buffer (which will be lossless little endian bytes regardless of type), and use a NullsBuffer to check for nulls.
##########
rust/sedona-raster-functions/src/crs_utils.rs:
##########
@@ -17,11 +17,32 @@
use std::borrow::Cow;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError, Result};
+use sedona_common::option::SedonaOptions;
use sedona_geometry::transform::{transform, CrsEngine};
+use sedona_proj::transform::with_global_proj_engine;
use sedona_schema::crs::{deserialize_crs, CoordinateReferenceSystem, Crs, CrsRef};
use wkb::reader::read_wkb;
+/// Run `f` with the session's CRS engine: the [`SedonaOptions`] runtime engine
+/// when config options are available (the query path), falling back to the
+/// process-global PROJ engine otherwise (e.g. direct `invoke_batch` calls).
+pub fn with_crs_engine<T>(
Review Comment:
I assume the direct calls are coming from the SedonaUDF tester or tests generally, which would use a default config options. We should ensure our testing config options have `SedonaOptions` in them (I think this is easy) and perhaps a `tester.with_crs_engine()` so that UDFs that use that particular aspect can add `sedona-proj` as a `dev-dependencies` and populate it for their tests.
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1435 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::ArrayRef;
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs, with_crs_engine};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_dataset_provider::configure_thread_local_options;
+use sedona_raster::traits::nodata_f64_to_bytes;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = RasterExecutor::num_iterations_over(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_crs_engine(config_options, |engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array.
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ RasterExecutor::finish_over(args, out)
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
Review Comment:
Here, and in a few other places, there's an allocation of a `Vec<u8>` of the size of the input that isn't reused. If possible we should allocate one `Vec<u8>` as scratch space and reuse it for all clip operations in a single batch. I'm not sure if there's an opportunity to save the final output and wrap it as a view that we append to the builder...I didn't see that here but I may have missed it.
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1435 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::ArrayRef;
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs, with_crs_engine};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_dataset_provider::configure_thread_local_options;
+use sedona_raster::traits::nodata_f64_to_bytes;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = RasterExecutor::num_iterations_over(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_crs_engine(config_options, |engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array.
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ RasterExecutor::finish_over(args, out)
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // nodata precedence: the explicit argument, then the band's own nodata
+ // bytes (used verbatim — no lossy f64 round-trip for Int64/UInt64),
+ // then the band data type's minimum value. An explicit value that is
+ // not exactly representable in the band's data type (fractional for an
+ // integer band, out of range, or a 64-bit integer beyond 2^53) errors
+ // rather than silently saturating — e.g. -9999 on UInt8 would collide
+ // with real zero-adjacent data.
+ let nodata_bytes: Vec<u8> = match custom_nodata {
+ Some(cn) => nodata_f64_to_bytes(cn, &data_type).map_err(|e| {
+ exec_datafusion_err!("RS_Clip: invalid no_data_value for band {band_idx}: {e}")
+ })?,
+ None => match band_metadata.nodata_value() {
+ Some(bytes) => bytes.to_vec(),
+ None => data_type.min_value_le_bytes(),
+ },
+ };
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, &nodata_bytes, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ &nodata_bytes,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_bytes,
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+
+ let col0 = min_col.floor();
+ let row0 = min_row.floor();
+ let col1 = max_col.ceil().max(col0 + 1.0);
+ let row1 = max_row.ceil().max(row0 + 1.0);
+
+ // Intersect with the raster extent. `>=` also rejects the NaN envelope of
+ // an empty geometry.
+ let col0 = col0.max(0.0);
+ let row0 = row0.max(0.0);
+ let col1 = col1.min(width as f64);
+ let row1 = row1.min(height as f64);
+ if !(col0 < col1 && row0 < row1) {
+ return Ok(None);
+ }
+
+ Ok(Some(CropWindow {
+ col_off: col0 as usize,
+ row_off: row0 as usize,
+ width: (col1 - col0) as usize,
+ height: (row1 - row0) as usize,
+ }))
+}
+
+/// Apply mask to band data (no cropping — preserves original dimensions).
+/// The mask covers only `window`; every pixel outside it is outside the
+/// geometry's envelope and therefore nodata.
+fn apply_mask_to_band(
+ original_data: &[u8],
+ mask: &[u8],
+ width: usize,
+ height: usize,
+ data_type: &BandDataType,
+ nodata_bytes: &[u8],
+ window: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let nodata_row: Vec<u8> = nodata_bytes.repeat(width);
+ let mut result = vec![0u8; width * height * byte_size];
+
+ let row_bytes = width * byte_size;
+ for row in 0..height {
+ let dst_row = &mut result[row * row_bytes..(row + 1) * row_bytes];
+ if row < window.row_off || row >= window.row_off + window.height {
+ dst_row.copy_from_slice(&nodata_row);
+ continue;
+ }
+ // Within a window row: nodata left and right of the window, source
+ // bytes inside it (masked-out pixels overwritten below).
+ let win_start = window.col_off * byte_size;
+ let win_end = (window.col_off + window.width) * byte_size;
+ dst_row[..win_start].copy_from_slice(&nodata_row[..win_start]);
+ dst_row[win_end..].copy_from_slice(&nodata_row[win_end..]);
+ let src_start = row * row_bytes + win_start;
+ dst_row[win_start..win_end]
+ .copy_from_slice(&original_data[src_start..src_start + (win_end - win_start)]);
+
+ let mask_row_start = (row - window.row_off) * window.width;
+ for col in 0..window.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = win_start + col * byte_size;
+ dst_row[dst..dst + byte_size].copy_from_slice(nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Apply mask AND crop to the given crop window in one pass. The mask is
+/// window-sized (row-major over `cw.width`×`cw.height`); the source data is
+/// the full raster plane.
+fn apply_mask_and_crop(
+ original_data: &[u8],
+ mask: &[u8],
+ full_width: usize,
+ data_type: &BandDataType,
+ nodata_bytes: &[u8],
+ cw: &CropWindow,
+) -> Result<Vec<u8>> {
+ let byte_size = data_type.byte_size();
+ let row_bytes = cw.width * byte_size;
+ let mut result = vec![0u8; cw.height * row_bytes];
+
+ // The crop-window columns of a source row are contiguous, so copy each row
+ // in one bulk memcpy (which vectorizes) rather than per pixel, then overwrite
+ // only the masked-out pixels with nodata. Masked pixels are written twice —
+ // once by the bulk copy, once here — but the bulk memcpy is far cheaper than
+ // the dynamic-width per-pixel copy it replaces.
+ for row in 0..cw.height {
+ let src_row = cw.row_off + row;
+ let src_start = (src_row * full_width + cw.col_off) * byte_size;
+ let dst_start = row * row_bytes;
+ result[dst_start..dst_start + row_bytes]
+ .copy_from_slice(&original_data[src_start..src_start + row_bytes]);
+
+ let mask_row_start = row * cw.width;
+ for col in 0..cw.width {
+ if mask[mask_row_start + col] == 0 {
+ let dst = dst_start + col * byte_size;
+ result[dst..dst + byte_size].copy_from_slice(nodata_bytes);
+ }
+ }
+ }
+
+ Ok(result)
+}
+
+/// Build the clipped raster via the N-D builder. A 2-D raster is just the
+/// `["y", "x"]` case; an N-D raster keeps its non-spatial dims and only its
+/// `(y, x)` extent changes when cropping.
+fn build_clipped_raster(
+ builder: &mut RasterBuilder,
+ original_raster: &RasterRefImpl<'_>,
+ clipped_data: &ClippedRasterData,
+) -> Result<()> {
+ // Geotransform is 2-D and shared across all planes. A crop shifts the
+ // upper-left by the pixel offset; scale/skew are unchanged.
+ // Layout: [upper_left_x, scale_x, skew_x, upper_left_y, skew_y, scale_y].
+ let src = original_raster.transform();
+ let transform: [f64; 6] = if let Some(cw) = clipped_data.crop_window {
+ let new_ulx = src[0] + cw.col_off as f64 * src[1] + cw.row_off as f64 * src[2];
+ let new_uly = src[3] + cw.row_off as f64 * src[5] + cw.col_off as f64 * src[4];
+ [new_ulx, src[1], src[2], new_uly, src[4], src[5]]
+ } else {
+ [src[0], src[1], src[2], src[3], src[4], src[5]]
+ };
Review Comment:
I'm still not sure how this works with rotated input but if there's a test that confirms it works I trust you
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1435 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::ArrayRef;
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::geo_transform::{GeoTransform, GeoTransformEx};
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_gdal::vector::geometry::Geometry;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs, with_crs_engine};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_dataset_provider::configure_thread_local_options;
+use sedona_raster::traits::nodata_f64_to_bytes;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = RasterExecutor::num_iterations_over(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_crs_engine(config_options, |engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array.
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ RasterExecutor::finish_over(args, out)
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height):
+ /// the geometry's envelope intersected with the raster extent, snapped
+ /// outward to the pixel grid. `None` means the full original raster
+ /// extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+
+ // The clip window is the geometry's envelope intersected with the raster
+ // extent, snapped outward to the pixel grid — the window PostGIS ST_Clip,
+ // `gdalwarp -crop_to_cutline`, and Sedona Spark's RS_Clip use. Knowing it
+ // up front rejects disjoint geometries before any GDAL work and bounds the
+ // mask to the window instead of the full raster.
+ let Some(window) = envelope_window(&geometry, &geotransform, width, height)? else {
+ return Ok(None);
+ };
+
+ // Create a mask raster covering only the clip window, with the geotransform
+ // shifted to the window's upper-left corner.
+ let mask_dataset =
+ MemDatasetBuilder::create(gdal, window.width, window.height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+ let (window_ulx, window_uly) = geotransform.apply(window.col_off as f64, window.row_off as f64);
+ let mask_geotransform = [
+ window_ulx,
+ geotransform[1],
+ geotransform[2],
+ window_uly,
+ geotransform[4],
+ geotransform[5],
+ ];
+ mask_dataset
+ .set_geo_transform(&mask_geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the (window-sized) mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>(
+ (0, 0),
+ (window.width, window.height),
+ (window.width, window.height),
+ None,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // The envelope may overlap the raster while the geometry itself selects no
+ // pixel (e.g. it falls between pixel centers); that is still the
+ // no-intersection case.
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ let crop_window = if crop { Some(window) } else { None };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // nodata precedence: the explicit argument, then the band's own nodata
+ // bytes (used verbatim — no lossy f64 round-trip for Int64/UInt64),
+ // then the band data type's minimum value. An explicit value that is
+ // not exactly representable in the band's data type (fractional for an
+ // integer band, out of range, or a 64-bit integer beyond 2^53) errors
+ // rather than silently saturating — e.g. -9999 on UInt8 would collide
+ // with real zero-adjacent data.
+ let nodata_bytes: Vec<u8> = match custom_nodata {
+ Some(cn) => nodata_f64_to_bytes(cn, &data_type).map_err(|e| {
+ exec_datafusion_err!("RS_Clip: invalid no_data_value for band {band_idx}: {e}")
+ })?,
+ None => match band_metadata.nodata_value() {
+ Some(bytes) => bytes.to_vec(),
+ None => data_type.min_value_le_bytes(),
+ },
+ };
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, &nodata_bytes, &cw)?
+ } else {
+ apply_mask_to_band(
+ plane_bytes,
+ mask,
+ width,
+ height,
+ &data_type,
+ &nodata_bytes,
+ &window,
+ )?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_bytes,
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the clip window: the geometry's envelope intersected with the
+/// raster extent, snapped outward to the pixel grid. Returns `None` when the
+/// envelope is disjoint from the raster extent (no clipping possible).
+///
+/// The envelope corners are mapped through the inverse geotransform (all four,
+/// so a skewed/rotated raster still gets a correct superset window) and the
+/// resulting pixel-space bbox is floored/ceiled to whole pixels. A degenerate
+/// envelope (point/line) landing exactly on a grid line is widened to one
+/// pixel so the rasterizer — not the snapping — decides whether it burns.
+fn envelope_window(
+ geometry: &Geometry,
+ geotransform: &GeoTransform,
+ width: usize,
+ height: usize,
+) -> Result<Option<CropWindow>> {
+ let env = geometry.envelope();
+ let inverse = geotransform
+ .invert()
+ .map_err(|e| exec_datafusion_err!("RS_Clip: geotransform is not invertible: {}", e))?;
+
+ let corners = [
+ (env.MinX, env.MinY),
+ (env.MinX, env.MaxY),
+ (env.MaxX, env.MinY),
+ (env.MaxX, env.MaxY),
+ ];
+ let mut min_col = f64::INFINITY;
+ let mut max_col = f64::NEG_INFINITY;
+ let mut min_row = f64::INFINITY;
+ let mut max_row = f64::NEG_INFINITY;
+ for (x, y) in corners {
+ let (col, row) = inverse.apply(x, y);
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
Review Comment:
Are you sure this logic survives a rotated and/or skewed raster? Is there a test that confirms this works?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3589969400
##########
python/sedonadb/tests/functions/raster_parity.py:
##########
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Helpers for validating RS_* functions against rasterio as a reference.
+
+The pattern: define a raster once as a numpy array + GDAL geotransform, write
+it to a GeoTIFF with rasterio, and run the function under test through both
+engines — sedonadb reads the file back via ``RS_FromPath`` and rasterio reads
+it directly — then compare pixel bytes, geotransform, and nodata exactly.
+Fixtures carry no CRS so neither engine reprojects and results stay
+bit-comparable.
+
+rasterio is imported lazily inside each helper so importing this module does
+not require it; test modules gate on ``pytest.importorskip("rasterio")``.
+"""
+
+import struct
+
+import numpy as np
+import pyarrow as pa
+
+from sedonadb.raster import BAND_DATA_TYPE_STRUCT_CHARS, BAND_DATA_TYPE_IDS, Raster
+
+
+def dtype_min(dtype) -> float:
+ """The minimum representable value of a numpy dtype — sedonadb's default
+ nodata sentinel when neither an explicit value nor a band nodata exists."""
+ dtype = np.dtype(dtype)
+ if dtype.kind == "f":
+ return float(np.finfo(dtype).min)
+ return int(np.iinfo(dtype).min)
+
+
+def write_geotiff(path, data: np.ndarray, *, gdal_transform, nodata=None) -> None:
+ """Write a ``(bands, height, width)`` array as a CRS-less GeoTIFF.
+
+ ``gdal_transform`` is GDAL-order ``(origin_x, scale_x, skew_x, origin_y,
+ skew_y, scale_y)``; ``nodata`` (optional) becomes the per-band nodata of
+ every band.
+ """
+ import rasterio
+ from rasterio.transform import Affine
+
+ bands, height, width = data.shape
+ with rasterio.open(
+ str(path),
+ "w",
+ driver="GTiff",
+ height=height,
+ width=width,
+ count=bands,
+ dtype=str(data.dtype),
+ transform=Affine.from_gdal(*gdal_transform),
+ nodata=nodata,
+ ) as dst:
+ dst.write(data)
+
+
+def rasterio_clip(path, geom_wkt: str, *, all_touched, nodata, crop):
+ """Reference clip composed from rasterio primitives over a GeoTIFF.
+
+ The window is ``rasterio.features.geometry_window`` (geometry bounds ∩
+ raster extent, snapped outward to the grid), the selection is
+ ``rasterio.features.geometry_mask``, and pixels compose as "inside the
+ geometry: source value verbatim; outside: ``nodata``". This is
+ deliberately not ``rasterio.mask.mask``, which additionally reads the
+ source masked and so remaps pixels *valued* at the band nodata to the
+ output ``nodata`` — a policy choice about source-nodata pixels, separate
+ from the geometry selection being validated here.
+
+ ``nodata`` must be the already-resolved fill value (the caller applies the
+ explicit-argument / band-nodata / dtype-minimum precedence). Returns
+ ``(array(bands, h, w), gdal_transform)``.
+
+ One caveat on independence: both engines ultimately burn the geometry with
+ GDAL's rasterizer (sedonadb via the system GDAL, rasterio via its bundled
+ one), so the pixel-center/all_touched selection rule itself is shared. The
+ genuinely independent implementations under comparison are everything on
+ top of the burn: window snapping, crop copy, transform shift, nodata
+ precedence, and band handling.
+ """
+ import rasterio
+ import rasterio.features
+ import rasterio.windows
+ import shapely
+
+ geom = shapely.from_wkt(geom_wkt)
+ with rasterio.open(str(path)) as src:
+ if crop:
+ window = rasterio.features.geometry_window(src, [geom])
+ else:
+ window = rasterio.windows.Window(0, 0, src.width, src.height)
+ transform = src.window_transform(window)
+ data = src.read(window=window)
+ inside = rasterio.features.geometry_mask(
+ [geom],
+ out_shape=(data.shape[1], data.shape[2]),
+ transform=transform,
+ all_touched=all_touched,
+ invert=True,
+ )
+ fill = np.asarray(nodata, dtype=data.dtype)
+ out = np.where(inside, data, fill)
+ return out, tuple(transform.to_gdal())
+
+
+def band_nodata(band):
+ """A band's nodata sentinel unpacked in its dtype, or None."""
+ raw = band._py_field("nodata")
+ if raw is None:
+ return None
+ type_id = BAND_DATA_TYPE_IDS[band.data_type]
+ return struct.unpack("<" + BAND_DATA_TYPE_STRUCT_CHARS[type_id], raw)[0]
+
+
+def raster_to_numpy(raster: Raster):
+ """Extract ``(array(bands, h, w), gdal_transform, [nodata per band])``."""
Review Comment:
done. made the 1-band case zero copy but multiband cases copy like you describe
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis merged PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "james-willis (via GitHub)" <gi...@apache.org>.
james-willis commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3582415589
##########
docs/reference/sql/rs_clip.qmd:
##########
@@ -0,0 +1,131 @@
+---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+title: RS_Clip
+description: >
+ Clips a raster to a geometry, setting pixels outside the geometry to nodata
+ and (by default) cropping to the geometry's bounding box.
+kernels:
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - {name: no_data_value, type: double}
+ - returns: raster
+ args:
+ - raster
+ - {name: band, type: integer}
+ - {name: geom, type: geometry}
+ - {name: all_touched, type: boolean}
+ - {name: no_data_value, type: double}
+ - {name: crop, type: boolean}
+ - returns: raster
+ args:
+ - raster
+ - name: band
+ type: integer
+ description: >
+ 1-based index of the band to clip, or 0 to clip every band. A NULL
+ band yields a NULL result.
+ - name: geom
+ type: geometry
+ description: >
+ Clip geometry. Reprojected into the raster's CRS when both carry one;
+ it is an error for exactly one side to have a CRS.
+ - name: all_touched
+ type: boolean
+ description: >
+ If true, keep every pixel the geometry touches; otherwise keep only
+ pixels whose center falls inside it. Defaults to false.
+ - name: no_data_value
+ type: double
+ description: >
+ Value written to pixels outside the geometry. Defaults to the band's
+ own nodata value, or the band data type's minimum if it has none. It is
+ an error if the value does not fit the band's data type.
+ - name: crop
+ type: boolean
+ description: Crop the output to the geometry's bounding box. Defaults to true.
+ - name: lenient
+ type: boolean
+ description: >
+ If true, a geometry that does not intersect the raster yields NULL;
+ if false, it raises an error. Defaults to true.
+---
+
+## Description
+
+`RS_Clip` clips a raster to a geometry, in the spirit of PostGIS `ST_Clip`.
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(rust/sedona-raster-gdal): add RS_Clip [sedona-db]
Posted by "prantogg (via GitHub)" <gi...@apache.org>.
prantogg commented on code in PR #1000:
URL: https://github.com/apache/sedona-db/pull/1000#discussion_r3546660597
##########
rust/sedona-raster-gdal/src/rs_clip.rs:
##########
@@ -0,0 +1,1367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! RS_Clip UDF - Clip a raster to a geometry boundary
+//!
+//! Similar to PostGIS ST_Clip, this function clips a raster to the extent of a geometry.
+//! Pixels outside the geometry are set to a nodata value: the explicit `no_data_value`
+//! argument if given, otherwise the band's own nodata value, otherwise the minimum value
+//! of the band's data type (so masked pixels stay distinguishable from real data).
+
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef};
+use datafusion_common::cast::{as_boolean_array, as_float64_array, as_int32_array};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::error::Result;
+use datafusion_common::exec_err;
+use datafusion_common::{exec_datafusion_err, ScalarValue};
+use datafusion_expr::{ColumnarValue, Volatility};
+use sedona_common::sedona_internal_err;
+use sedona_gdal::gdal::Gdal;
+use sedona_gdal::mem::MemDatasetBuilder;
+use sedona_gdal::raster::types::GdalDataType;
+use sedona_proj::transform::with_global_proj_engine;
+
+use arrow_schema::DataType;
+use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
+use sedona_raster::array::RasterRefImpl;
+use sedona_raster::builder::RasterBuilder;
+use sedona_raster::traits::{is_spatial_dim_pair, RasterRef};
+use sedona_raster_functions::crs_utils::{crs_transform_wkb, resolve_crs};
+use sedona_raster_functions::rs_ensure_loaded::{
+ NEEDS_PIXELS_METADATA_KEY, RETURNS_BYTES_METADATA_KEY,
+};
+use sedona_raster_functions::RasterExecutor;
+use sedona_schema::datatypes::{SedonaType, RASTER};
+use sedona_schema::matchers::ArgMatcher;
+use sedona_schema::raster::BandDataType;
+
+use crate::gdal_common::with_gdal;
+use crate::gdal_common::{nodata_bytes_to_f64, nodata_f64_to_bytes};
+use crate::gdal_dataset_provider::configure_thread_local_options;
+
+/// RS_Clip() scalar UDF implementation
+///
+/// Clips a raster to a geometry boundary.
+///
+/// Signatures:
+/// - `RS_Clip(raster, band, geom)` — 3 args
+/// - `RS_Clip(raster, band, geom, allTouched)` — 4 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue)` — 5 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop)` — 6 args
+/// - `RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)` — 7 args
+pub fn rs_clip_udf() -> SedonaScalarUDF {
+ SedonaScalarUDF::new(
+ "rs_clip",
+ vec![
+ Arc::new(RsClip { arg_count: 3 }), // (raster, band, geom)
+ Arc::new(RsClip { arg_count: 4 }), // (raster, band, geom, allTouched)
+ Arc::new(RsClip { arg_count: 5 }), // (raster, band, geom, allTouched, noDataValue)
+ Arc::new(RsClip { arg_count: 6 }), // (raster, band, geom, allTouched, noDataValue, crop)
+ Arc::new(RsClip { arg_count: 7 }), // (raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ],
+ Volatility::Immutable,
+ )
+ // Reads band pixels (so the planner materializes OutDb rasters via
+ // RS_EnsureLoaded first) and emits a fresh InDb raster (so its output is
+ // already loaded and isn't wrapped again).
+ .with_metadata(NEEDS_PIXELS_METADATA_KEY, "true")
+ .with_metadata(RETURNS_BYTES_METADATA_KEY, "true")
+}
+
+/// Kernel implementation for RS_Clip
+#[derive(Debug)]
+struct RsClip {
+ /// Number of arguments in the matched signature (3..=7)
+ arg_count: usize,
+}
+
+impl SedonaScalarKernel for RsClip {
+ fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> {
+ let matchers = match self.arg_count {
+ 3 => vec![
+ // RS_Clip(raster, band, geom)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ],
+ 4 => vec![
+ // RS_Clip(raster, band, geom, allTouched)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ],
+ 5 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ],
+ 6 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ],
+ 7 => vec![
+ // RS_Clip(raster, band, geom, allTouched, noDataValue, crop, lenient)
+ ArgMatcher::is_raster(),
+ ArgMatcher::is_integer(),
+ ArgMatcher::is_geometry_or_geography(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_numeric(),
+ ArgMatcher::is_boolean(),
+ ArgMatcher::is_boolean(),
+ ],
+ _ => {
+ return sedona_internal_err!("RS_Clip: unexpected arg_count {}", self.arg_count);
+ }
+ };
+
+ let matcher = ArgMatcher::new(matchers, RASTER);
+ matcher.match_args(args)
+ }
+
+ fn invoke_batch(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ ) -> Result<ColumnarValue> {
+ self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None)
+ }
+
+ fn invoke_batch_from_args(
+ &self,
+ arg_types: &[SedonaType],
+ args: &[ColumnarValue],
+ _return_type: &SedonaType,
+ _num_rows: usize,
+ config_options: Option<&ConfigOptions>,
+ ) -> Result<ColumnarValue> {
+ let num_iterations = calc_num_iterations(args);
+
+ // Band is always at index 1, geom is always at index 2.
+ let geom_arg_idx: usize = 2;
+
+ // Expand band to array
+ let band_array = args[1]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Int32, None)?
+ .into_array(num_iterations)?;
+ let band_array = as_int32_array(&band_array)?.clone();
+
+ // allTouched at index 3 (when arg_count >= 4)
+ let all_touched_array = if self.arg_count >= 4 {
+ args[3]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(false)).to_array_of_size(num_iterations)?
+ };
+ let all_touched_array = as_boolean_array(&all_touched_array)?.clone();
+
+ // noDataValue at index 4 (when arg_count >= 5)
+ let nodata_array = if self.arg_count >= 5 {
+ args[4]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Float64, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Float64(None).to_array_of_size(num_iterations)?
+ };
+ let nodata_array = as_float64_array(&nodata_array)?.clone();
+
+ // crop at index 5 (when arg_count >= 6), default true
+ let crop_array = if self.arg_count >= 6 {
+ args[5]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let crop_array = as_boolean_array(&crop_array)?.clone();
+
+ // lenient at index 6 (when arg_count >= 7), default true
+ let lenient_array = if self.arg_count >= 7 {
+ args[6]
+ .clone()
+ .cast_to(&arrow_schema::DataType::Boolean, None)?
+ .into_array(num_iterations)?
+ } else {
+ ScalarValue::Boolean(Some(true)).to_array_of_size(num_iterations)?
+ };
+ let lenient_array = as_boolean_array(&lenient_array)?.clone();
+
+ let mut band_iter = band_array.iter();
+ let mut all_touched_iter = all_touched_array.iter();
+ let mut nodata_iter = nodata_array.iter();
+ let mut crop_iter = crop_array.iter();
+ let mut lenient_iter = lenient_array.iter();
+
+ // Build output rasters
+ let mut builder = RasterBuilder::new(num_iterations);
+
+ let exec_arg_types = vec![arg_types[0].clone(), arg_types[geom_arg_idx].clone()];
+ let exec_args = vec![args[0].clone(), args[geom_arg_idx].clone()];
+ let executor =
+ RasterExecutor::new_with_num_iterations(&exec_arg_types, &exec_args, num_iterations);
+
+ with_gdal(|gdal| {
+ configure_thread_local_options(gdal, config_options)?;
+ with_global_proj_engine(|engine| {
+ executor.execute_raster_wkb_crs_void(|raster_opt, wkb_opt, geom_crs| {
+ // Advance every option iterator in lockstep. A NULL in the band
+ // or any boolean flag propagates to a NULL output row (SQL
+ // semantics) — in particular a NULL band must NOT fall through
+ // to the "band 0 = all bands" mode. `no_data_value` is the
+ // exception: its NULL is the meaningful "not supplied" sentinel
+ // (the 3/4-arg signatures pass NULL here), so it stays an Option.
+ let band = band_iter.next().flatten();
+ let all_touched = all_touched_iter.next().flatten();
+ let nodata_value = nodata_iter.next().flatten();
+ let crop = crop_iter.next().flatten();
+ let lenient = lenient_iter.next().flatten();
+ let (Some(band), Some(all_touched), Some(crop), Some(lenient)) =
+ (band, all_touched, crop, lenient)
+ else {
+ builder.append_null()?;
+ return Ok(());
+ };
+
+ let (raster, geom_wkb) = match (raster_opt, wkb_opt) {
+ (Some(r), Some(w)) => (r, w),
+ _ => {
+ builder.append_null()?;
+ return Ok(());
+ }
+ };
+
+ let raster_crs = resolve_crs(raster.crs())?;
+ let geom_wkb = match (geom_crs, raster_crs.as_deref()) {
+ (Some(geom_crs), Some(raster_crs)) => {
+ crs_transform_wkb(geom_wkb, geom_crs, raster_crs, engine)?
+ }
+ (None, None) => geom_wkb.to_vec(),
+ (Some(_), None) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: raster has no CRS but geometry does"
+ )
+ }
+ (None, Some(_)) => {
+ return exec_err!(
+ "Cannot operate on geometry and raster: geometry has no CRS but raster does"
+ )
+ }
+ };
+
+ // Band 0 means "all bands" (handled in clip_raster, which also
+ // range-checks the upper bound). A negative band is an error,
+ // not a silent clamp to band 1.
+ if band < 0 {
+ return exec_err!(
+ "RS_Clip: band must be >= 0 (0 = all bands), got {band}"
+ );
+ }
+ let band_index = band as usize;
+ match clip_raster(
+ gdal,
+ raster,
+ &geom_wkb,
+ band_index,
+ nodata_value,
+ all_touched,
+ crop,
+ ) {
+ Ok(Some(clipped_data)) => {
+ build_clipped_raster(&mut builder, raster, &clipped_data)?
+ }
+ Ok(None) => {
+ // The clip mask is empty — no pixel was selected. `lenient`
+ // yields NULL either way; strict errors, with a message
+ // conditioned on `all_touched`: when it is already true an
+ // empty mask means the geometry is genuinely disjoint,
+ // whereas with the default `all_touched = false` the
+ // geometry may still overlap but fall between pixel centers.
+ if lenient {
+ builder.append_null()?;
+ } else if all_touched {
+ return exec_err!("RS_Clip: raster and geometry do not intersect");
+ } else {
+ return exec_err!(
+ "RS_Clip: geometry selects no pixels; it may fall between \
+ pixel centers — pass all_touched => true to keep any pixel it touches"
+ );
+ }
+ }
+ // A genuine failure (malformed WKB, GDAL error, …) always
+ // propagates — it is not the no-intersection case `lenient`
+ // is meant to soften.
+ Err(e) => return Err(e),
+ }
+
+ Ok(())
+ })
+ })?;
+
+ // Decide array-vs-scalar over *all* args, not just the raster/geom
+ // the executor was given: a per-row band/option column over a scalar
+ // raster+geom must still yield an N-row array. (`executor.finish`
+ // only inspects its two exec args and would collapse to row 0.)
+ let out: ArrayRef = Arc::new(builder.finish()?);
+ if args.iter().any(|a| matches!(a, ColumnarValue::Array(_))) {
+ Ok(ColumnarValue::Array(out))
+ } else {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(&out, 0)?))
+ }
+ })
+ }
+}
+
+/// One clipped band: masked/cropped bytes plus the N-D layout needed to rebuild
+/// it. The clip is a 2-D `(y, x)` operation broadcast across every non-spatial
+/// plane, so `dim_names` are unchanged from the source and only the trailing
+/// `(y, x)` extent of `shape` shrinks when cropping.
+struct ClippedBand {
+ /// Masked/cropped bytes, plane-major in the band's dim order.
+ data: Vec<u8>,
+ /// Visible dim names, unchanged from the source band (e.g. `["time","y","x"]`).
+ dim_names: Vec<String>,
+ /// Output shape: leading non-spatial dims unchanged, trailing `(y, x)` clipped.
+ shape: Vec<i64>,
+ data_type: BandDataType,
+ /// nodata sentinel bytes written for masked-out pixels.
+ nodata: Vec<u8>,
+ /// Source band name, preserved so clipping a named/N-D band keeps it
+ /// addressable by name on the output.
+ name: Option<String>,
+}
+
+/// Data for a clipped raster
+struct ClippedRasterData {
+ /// One entry per processed band.
+ bands: Vec<ClippedBand>,
+ /// Crop window in pixel coordinates (col_off, row_off, width, height).
+ /// `None` means the full original raster extent was kept (crop=false).
+ crop_window: Option<CropWindow>,
+}
+
+/// A rectangular crop window in pixel coordinates.
+#[derive(Debug, Clone, Copy)]
+struct CropWindow {
+ col_off: usize,
+ row_off: usize,
+ width: usize,
+ height: usize,
+}
+
+/// Clip a raster to a geometry.
+///
+/// Returns `Ok(None)` when the geometry does not intersect the raster extent
+/// (caller decides how to handle based on `lenient`).
+fn clip_raster(
+ gdal: &Gdal,
+ raster: &RasterRefImpl<'_>,
+ geom_wkb: &[u8],
+ band_num: usize,
+ custom_nodata: Option<f64>,
+ all_touched: bool,
+ crop: bool,
+) -> Result<Option<ClippedRasterData>> {
+ let metadata = raster.metadata();
+ let bands = raster.bands();
+ let width = metadata.width() as usize;
+ let height = metadata.height() as usize;
+
+ // Parse geometry from WKB
+ let geometry = gdal
+ .geometry_from_wkb(geom_wkb)
+ .map_err(|e| exec_datafusion_err!("Failed to parse geometry from WKB: {}", e))?;
+
+ // Create a mask raster (same dimensions as input)
+ let mask_dataset = MemDatasetBuilder::create(gdal, width, height, 1, GdalDataType::UInt8)
+ .map_err(|e| exec_datafusion_err!("Failed to create mask dataset: {}", e))?;
+
+ // Set the same geotransform as the input raster
+ let geotransform = [
+ metadata.upper_left_x(),
+ metadata.scale_x(),
+ metadata.skew_x(),
+ metadata.upper_left_y(),
+ metadata.skew_y(),
+ metadata.scale_y(),
+ ];
+ mask_dataset
+ .set_geo_transform(&geotransform)
+ .map_err(|e| exec_datafusion_err!("Failed to set geotransform: {}", e))?;
+
+ // GDAL's MEM driver zero-fills owned band buffers at creation, so the mask
+ // already reads 0 (outside) everywhere; rasterize_affine burns 1 inside the
+ // geometry. No explicit zero-init write needed.
+ gdal.rasterize_affine(
+ &mask_dataset,
+ &[1], // band 1
+ &[geometry],
+ &[1.0], // burn value = 1 (inside)
+ all_touched,
+ )
+ .map_err(|e| exec_datafusion_err!("Failed to rasterize geometry: {}", e))?;
+
+ // Read the mask
+ let mask_band = mask_dataset
+ .rasterband(1)
+ .map_err(|e| exec_datafusion_err!("Failed to get mask band: {}", e))?;
+ let mask_buffer = mask_band
+ .read_as::<u8>((0, 0), (width, height), (width, height), None)
+ .map_err(|e| exec_datafusion_err!("Failed to read mask: {}", e))?;
+ let mask = mask_buffer.data();
+
+ // Check if there are any non-zero pixels in the mask (i.e. geometry intersects raster)
+ let has_intersection = mask.iter().any(|&v| v != 0);
+ if !has_intersection {
+ return Ok(None);
+ }
+
+ // Compute crop window if crop=true
+ let crop_window = if crop {
+ compute_crop_window(mask, width, height)
+ } else {
+ None
+ };
+
+ // Determine which bands to process
+ let band_indices: Vec<usize> = if band_num == 0 {
+ (1..=bands.len()).collect()
+ } else {
+ if band_num > bands.len() {
+ return exec_err!("Band {} is out of range (1-{})", band_num, bands.len());
+ }
+ vec![band_num]
+ };
+
+ // Process each band. The clip is a 2-D (y, x) operation; for an N-D band
+ // (extra leading dims such as time) the same mask and crop window are
+ // broadcast across every non-spatial plane.
+ let mut clipped_bands = Vec::with_capacity(band_indices.len());
+
+ for &band_idx in &band_indices {
+ let band = bands
+ .band(band_idx)
+ .map_err(|e| exec_datafusion_err!("Failed to get band {}: {}", band_idx, e))?;
+ // `band_idx` is 1-based; the `band_name` accessor is 0-based.
+ let band_name = raster.band_name(band_idx - 1).map(|s| s.to_string());
+
+ let band_metadata = band.metadata();
+ let data_type = band_metadata.data_type()?;
+
+ // The trailing two axes are the spatial (y, x) plane; anything before
+ // them is a stack of planes the 2-D clip is broadcast over.
+ let shape = band.shape().to_vec();
+ let dim_names: Vec<String> = band.dim_names().iter().map(|s| s.to_string()).collect();
+ let ndim = shape.len();
+ if ndim < 2 {
+ return exec_err!(
+ "RS_Clip: band {} has {} dimension(s); a 2-D (y, x) plane is required",
+ band_idx,
+ ndim
+ );
+ }
+ // The trailing two dims must actually be the spatial (y, x) pair. A band
+ // whose trailing dims are e.g. ["x","y"] is legal at the format level and
+ // would otherwise be masked/cropped transposed on a square raster; mirror
+ // the `is_spatial_dim_pair` guard the GDAL bridge (gdal_common) performs.
+ if !is_spatial_dim_pair(&dim_names[ndim - 2], &dim_names[ndim - 1]) {
+ return exec_err!(
+ "RS_Clip: band {} trailing dims {:?} are not a (y, x) spatial pair",
+ band_idx,
+ &dim_names[ndim - 2..]
+ );
+ }
+ let (plane_h, plane_w) = (shape[ndim - 2] as usize, shape[ndim - 1] as usize);
+ if plane_w != width || plane_h != height {
+ return exec_err!(
+ "RS_Clip: band {} spatial extent {}x{} does not match the raster {}x{}",
+ band_idx,
+ plane_w,
+ plane_h,
+ width,
+ height
+ );
+ }
+ let n_planes: usize = shape[..ndim - 2].iter().map(|&d| d as usize).product();
+
+ // `as_contiguous` borrows the band bytes; we only ever read them here
+ // (the mask/crop helpers allocate their own output), so no copy is needed.
+ let nd_buffer = band.nd_buffer().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: failed to read band {}: {}", band_idx, e)
+ })?;
+ let original_data = nd_buffer.as_contiguous().map_err(|e| {
+ exec_datafusion_err!("RS_Clip: band {} is not contiguous: {}", band_idx, e)
+ })?;
+
+ // A custom nodata that isn't representable in the band's data type would
+ // be silently saturated by `nodata_f64_to_bytes` (e.g. -9999 -> 0 on
+ // UInt8, colliding with real data), so reject it explicitly here.
+ if let Some(cn) = custom_nodata {
+ let (lo, hi) = (
+ band_data_type_min(&data_type),
+ band_data_type_max(&data_type),
+ );
+ if cn < lo || cn > hi {
+ return exec_err!(
+ "RS_Clip: no_data_value {cn} is outside the representable range [{lo}, {hi}] of band {band_idx}'s data type {data_type:?}"
+ );
+ }
+ }
+
+ // nodata precedence: the explicit argument, then the band's own nodata,
+ // then the band data type's minimum value — never a silent 0.0, which
+ // would be indistinguishable from real zero-valued pixels.
+ let nodata = custom_nodata
+ .or_else(|| nodata_bytes_to_f64(band_metadata.nodata_value(), &data_type))
+ .unwrap_or_else(|| band_data_type_min(&data_type));
+
+ let byte_size = data_type.byte_size();
+ let in_plane_bytes = width * height * byte_size;
+ if original_data.len() != n_planes * in_plane_bytes {
+ return exec_err!(
+ "RS_Clip: band {} byte length {} does not match {} planes of {}x{}",
+ band_idx,
+ original_data.len(),
+ n_planes,
+ width,
+ height
+ );
+ }
+
+ // Apply the (shared) mask/crop to each plane, then concatenate.
+ let out_plane_bytes = match crop_window {
+ Some(cw) => cw.width * cw.height * byte_size,
+ None => in_plane_bytes,
+ };
+ let mut clipped_data = Vec::with_capacity(n_planes * out_plane_bytes);
+ for plane in 0..n_planes {
+ let plane_bytes = &original_data[plane * in_plane_bytes..(plane + 1) * in_plane_bytes];
+ let clipped_plane = if let Some(cw) = crop_window {
+ apply_mask_and_crop(plane_bytes, mask, width, &data_type, nodata, &cw)?
+ } else {
+ apply_mask_to_band(plane_bytes, mask, width, height, &data_type, nodata)?
+ };
+ clipped_data.extend_from_slice(&clipped_plane);
+ }
+
+ // Output shape: leading dims unchanged; trailing (y, x) becomes the crop
+ // window when cropping, else the original plane extent.
+ let (out_h, out_w) = match crop_window {
+ Some(cw) => (cw.height as i64, cw.width as i64),
+ None => (height as i64, width as i64),
+ };
+ let mut out_shape = shape[..ndim - 2].to_vec();
+ out_shape.push(out_h);
+ out_shape.push(out_w);
+
+ clipped_bands.push(ClippedBand {
+ data: clipped_data,
+ dim_names,
+ shape: out_shape,
+ data_type,
+ nodata: nodata_f64_to_bytes(nodata, &data_type),
+ name: band_name,
+ });
+ }
+
+ Ok(Some(ClippedRasterData {
+ bands: clipped_bands,
+ crop_window,
+ }))
+}
+
+/// Compute the minimal bounding pixel window that contains all non-zero mask pixels.
+fn compute_crop_window(mask: &[u8], width: usize, height: usize) -> Option<CropWindow> {
+ let mut min_col = width;
+ let mut max_col = 0usize;
+ let mut min_row = height;
+ let mut max_row = 0usize;
+
+ for row in 0..height {
+ for col in 0..width {
+ if mask[row * width + col] != 0 {
+ min_col = min_col.min(col);
+ max_col = max_col.max(col);
+ min_row = min_row.min(row);
+ max_row = max_row.max(row);
+ }
+ }
+ }
+
+ if min_col > max_col || min_row > max_row {
+ return None; // no non-zero pixels (shouldn't happen if caller checked)
+ }
+
+ Some(CropWindow {
+ col_off: min_col,
+ row_off: min_row,
+ width: max_col - min_col + 1,
+ height: max_row - min_row + 1,
+ })
+}
Review Comment:
There's a performance angle too. Tight crop means the output extent isn't known until after the mask is burned, so every clip pays for a full-raster W×H mask (MEM dataset + the full scans in `has_intersection` and here) even when the geometry covers a handful of pixels.
Envelope crop knows the window up front, which enables three wins:
- the mask shrinks to the window
- this scan goes away
- and disjoint geometries can be rejected with a cheap envelope check before any GDAL work
Sedona Spark's implementation is structured exactly this way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@sedona.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org