You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pr...@apache.org on 2026/06/02 23:35:49 UTC
(sedona-spatialbench) branch pranav/feature/raster-benchmark updated: feat(raster): optional target_compression_ratio, calibrate frequency to it
This is an automated email from the ASF dual-hosted git repository.
prantogg pushed a commit to branch pranav/feature/raster-benchmark
in repository https://gitbox.apache.org/repos/asf/sedona-spatialbench.git
The following commit(s) were added to refs/heads/pranav/feature/raster-benchmark by this push:
new 554ef03 feat(raster): optional target_compression_ratio, calibrate frequency to it
554ef03 is described below
commit 554ef03b158d7f6c1e7dd1a5ab297a9f6a9e9306
Author: Pranav Toggi <pr...@gmail.com>
AuthorDate: Tue Jun 2 16:33:51 2026 -0700
feat(raster): optional target_compression_ratio, calibrate frequency to it
Compression ratio is the user-meaningful intent (real-world engine I/O load);
noise_frequency is a mechanism whose ratio is dimension- and dtype-specific
(112 only yields ~2x at 10980² UInt16). Add an optional target_compression_ratio
that calibrates noise_frequency at startup to hit it for the configured
dtype/dimensions/zstd_level, removing trial-and-error and the silent footgun of
a fixed frequency that breaks when dimensions/dtype change.
Calibration walks the descending (smooth-noise) arm of ratio-vs-frequency with a
coarse geometric sweep, then bisects within the bracketed interval. (Ratio is
NOT globally monotonic: it falls as detail rises, then rises again from aliasing
at high frequency — so the search is confined to the smooth arm, which is also
the realistic regime.) Deterministic: fixed seed, fixed sample tiles, fixed
sweep/bisection. zstd_level and noise_frequency remain low-level knobs; an
out-of-range target clamps to the arm end with a warning.
---
spatialbench-cli/src/main.rs | 27 +++-
spatialbench-cli/src/spatial_config_file.rs | 54 ++++++-
spatialbench-raster/src/cog/mod.rs | 220 ++++++++++++++++++++++++++++
3 files changed, 297 insertions(+), 4 deletions(-)
diff --git a/spatialbench-cli/src/main.rs b/spatialbench-cli/src/main.rs
index db71d04..82990ad 100644
--- a/spatialbench-cli/src/main.rs
+++ b/spatialbench-cli/src/main.rs
@@ -411,12 +411,37 @@ impl Cli {
let raster_config = parsed_config.as_ref().and_then(|c| c.raster.as_ref());
- let cog_config = if let Some(rc) = raster_config {
+ let mut cog_config = if let Some(rc) = raster_config {
rc.to_cog_config()?
} else {
CogConfig::default()
};
+ // If a target compression ratio is requested, calibrate the noise
+ // frequency to hit it (dtype/dimension-aware), overriding the
+ // configured/default noise_frequency.
+ if let Some(target) = raster_config.and_then(|rc| rc.target_compression_ratio) {
+ let cal = spatialbench_raster::cog::calibrate_frequency(&cog_config, target)?;
+ if cal.clamped {
+ log::warn!(
+ "target_compression_ratio={target} unreachable for dtype={:?} at {}x{}; \
+ clamped noise_frequency={:.2} (achieved ~{:.2}x)",
+ cog_config.dtype,
+ cog_config.raster.cog_width,
+ cog_config.raster.cog_height,
+ cal.frequency,
+ cal.achieved_ratio,
+ );
+ } else {
+ info!(
+ "calibrated noise_frequency={:.2} for target_compression_ratio={target} \
+ (achieved ~{:.2}x)",
+ cal.frequency, cal.achieved_ratio,
+ );
+ }
+ cog_config.noise_frequency = cal.frequency;
+ }
+
let continent_name = raster_config
.map(|rc| rc.continent.as_str())
.unwrap_or("south_north_america");
diff --git a/spatialbench-cli/src/spatial_config_file.rs b/spatialbench-cli/src/spatial_config_file.rs
index 6fc81e8..02df70f 100644
--- a/spatialbench-cli/src/spatial_config_file.rs
+++ b/spatialbench-cli/src/spatial_config_file.rs
@@ -117,9 +117,10 @@ pub struct RasterConfig {
/// Internal COG tile size (pixels per side).
#[serde(default = "default_tile_size")]
pub tile_size: u32,
- /// Perlin noise frequency. Primary knob for compression ratio: higher
- /// frequency = higher entropy = lower ratio. At 10980² UInt16, ~128 yields
- /// the ~2× ratio typical of real Sentinel-2 L2A 10m bands.
+ /// Perlin noise frequency. Low-level knob for compression ratio: higher
+ /// frequency = higher entropy = lower ratio. Dimension- and dtype-specific
+ /// (e.g. ~112 yields ~2× only at 10980² UInt16). Ignored when
+ /// `target_compression_ratio` is set.
#[serde(default = "default_noise_frequency")]
pub noise_frequency: f32,
/// Pixel data type: "uint8", "uint16", "float32".
@@ -128,6 +129,13 @@ pub struct RasterConfig {
/// ZSTD compression level (1=fastest .. 22). See [`CogConfig::zstd_level`].
#[serde(default = "default_zstd_level")]
pub zstd_level: i32,
+ /// Optional target compression ratio (raw / compressed, e.g. 2.0). When set,
+ /// `noise_frequency` is calibrated at startup to hit this ratio for the
+ /// configured dtype/dimensions/zstd_level, and the configured/default
+ /// `noise_frequency` is ignored. Must be > 1.0. Prefer this over hand-tuning
+ /// `noise_frequency`, which is dimension- and dtype-specific.
+ #[serde(default)]
+ pub target_compression_ratio: Option<f32>,
/// Continent for spatial coverage. Valid values:
/// "south_north_america" (default), "north_north_america", "europe",
/// "africa", "south_asia", "north_asia", "oceania", "south_america".
@@ -204,6 +212,7 @@ impl Default for RasterConfig {
noise_frequency: default_noise_frequency(),
dtype: default_dtype(),
zstd_level: default_zstd_level(),
+ target_compression_ratio: None,
continent: default_continent(),
}
}
@@ -211,8 +220,20 @@ impl Default for RasterConfig {
impl RasterConfig {
/// Convert to [`CogConfig`], parsing the dtype string.
+ ///
+ /// Note: `target_compression_ratio` is validated here but applied by the
+ /// caller (it requires a calibration pass over sample tiles), so this
+ /// conversion stays pure. `noise_frequency` is passed through as-is.
pub fn to_cog_config(&self) -> io::Result<CogConfig> {
let dtype = parse_dtype(&self.dtype)?;
+ if let Some(r) = self.target_compression_ratio {
+ if !(r.is_finite() && r > 1.0) {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidInput,
+ format!("target_compression_ratio must be a finite value > 1.0, got {r}"),
+ ));
+ }
+ }
Ok(CogConfig {
raster: FootprintConfig {
cog_width: self.cog_width,
@@ -404,4 +425,31 @@ raster:
let cfg = parse_yaml(yaml).unwrap();
assert!(cfg.raster.is_none());
}
+
+ #[test]
+ fn target_ratio_parses_when_present() {
+ let raster = parse_yaml("raster:\n target_compression_ratio: 2.5")
+ .unwrap()
+ .raster
+ .unwrap();
+ assert_eq!(raster.target_compression_ratio, Some(2.5));
+ }
+
+ #[test]
+ fn target_ratio_absent_is_none() {
+ let raster = parse_yaml("raster: {}").unwrap().raster.unwrap();
+ assert_eq!(raster.target_compression_ratio, None);
+ }
+
+ #[test]
+ fn target_ratio_invalid_rejected() {
+ for bad in ["1.0", "0.5", ".nan"] {
+ let raster = parse_yaml(&format!("raster:\n target_compression_ratio: {bad}"))
+ .unwrap()
+ .raster
+ .unwrap();
+ let err = raster.to_cog_config().unwrap_err();
+ assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
+ }
+ }
}
diff --git a/spatialbench-raster/src/cog/mod.rs b/spatialbench-raster/src/cog/mod.rs
index c659d16..9a24f96 100644
--- a/spatialbench-raster/src/cog/mod.rs
+++ b/spatialbench-raster/src/cog/mod.rs
@@ -508,6 +508,144 @@ fn write_geotiff_tags(
Ok(())
}
+/// Frequency search domain for ratio calibration.
+const CALIB_FREQ_MIN: f32 = 2.0;
+const CALIB_FREQ_MAX: f32 = 2048.0;
+/// Geometric step for the coarse sweep that maps the descending arm.
+const CALIB_SWEEP_STEP: f32 = 1.3;
+/// Sample an N×N block of full interior tiles per probe (cheap, representative).
+const CALIB_TILES: u32 = 4;
+/// Bisection iterations to refine within the bracketed sub-interval.
+const CALIB_ITERS: usize = 12;
+
+/// Result of calibrating `noise_frequency` to a target compression ratio.
+#[derive(Debug, Clone, Copy)]
+pub struct CalibrationResult {
+ /// Resolved frequency to use for generation.
+ pub frequency: f32,
+ /// Ratio actually achieved by the sample at `frequency`.
+ pub achieved_ratio: f32,
+ /// True if `target_ratio` was outside the achievable range and `frequency`
+ /// was clamped to the nearest end of the achievable (smooth-noise) arm.
+ pub clamped: bool,
+}
+
+/// Resolve the `noise_frequency` that yields ~`target_ratio` for this config's
+/// dtype, dimensions, tile size, and zstd level.
+///
+/// Ratio vs frequency is **not globally monotonic**: it falls as smooth-gradient
+/// detail rises, reaches a minimum, then rises again once the frequency is high
+/// enough that aliasing/periodicity sets in (and collapses entirely at
+/// integer-lattice frequencies, where Perlin is 0). Only the initial *descending
+/// arm* is genuine smooth-noise — the realistic regime. So we walk that arm with
+/// a coarse geometric sweep, stop when the ratio turns back up, and bisect within
+/// it. At realistic dimensions (e.g. 10980²) the arm spans the whole useful
+/// range (~1.1×–6×); small images have a shorter arm (higher minimum ratio), in
+/// which case an out-of-range target clamps to the arm end.
+///
+/// Deterministic: fixed seed (0), fixed sample tiles, fixed sweep/bisection —
+/// the same `(config, target_ratio)` always yields the same frequency. The
+/// caller should log the result. Cost is a few dozen small-tile compressions
+/// (sub-second), negligible vs a real run.
+///
+/// # Errors
+///
+/// Returns `io::Error` if ZSTD compression fails during a probe.
+pub fn calibrate_frequency(
+ config: &CogConfig,
+ target_ratio: f32,
+) -> io::Result<CalibrationResult> {
+ // Coarse geometric sweep along the descending arm, stopping as soon as we
+ // bracket the target (or leave the smooth arm / hit the cap). Stopping early
+ // keeps both startup and tests cheap.
+ let mut prev: Option<(f32, f32)> = None;
+ let mut freq = CALIB_FREQ_MIN;
+ loop {
+ let r = measure_ratio(config, freq)?;
+ match prev {
+ // First probe: if the target is already at/above the highest ratio,
+ // the lowest frequency is the best we can do.
+ None => {
+ if target_ratio >= r {
+ return Ok(CalibrationResult { frequency: freq, achieved_ratio: r, clamped: true });
+ }
+ }
+ Some((pf, pr)) => {
+ if r >= pr {
+ // Ratio turned back up → end of the smooth arm at `prev`, and
+ // the target is below the arm's minimum → clamp to the arm end.
+ return Ok(CalibrationResult { frequency: pf, achieved_ratio: pr, clamped: true });
+ }
+ if r <= target_ratio {
+ // Bracketed: ratio(pf) > target >= ratio(freq). Bisect within
+ // [pf, freq] — monotonic on the arm, so this is well-defined.
+ let (mut lo, mut hi) = (pf, freq);
+ let (mut mid, mut mid_ratio) = (freq, r);
+ for _ in 0..CALIB_ITERS {
+ mid = 0.5 * (lo + hi);
+ mid_ratio = measure_ratio(config, mid)?;
+ if mid_ratio > target_ratio {
+ lo = mid; // ratio too high → need more frequency
+ } else {
+ hi = mid; // ratio too low → back off frequency
+ }
+ }
+ return Ok(CalibrationResult { frequency: mid, achieved_ratio: mid_ratio, clamped: false });
+ }
+ }
+ }
+ if freq >= CALIB_FREQ_MAX {
+ // Hit the cap with ratio still above target → clamp to the cap.
+ return Ok(CalibrationResult { frequency: freq, achieved_ratio: r, clamped: true });
+ }
+ prev = Some((freq, r));
+ freq = (freq * CALIB_SWEEP_STEP).min(CALIB_FREQ_MAX);
+ }
+}
+
+/// Average compression ratio (raw / compressed) over a small block of full
+/// interior tiles, using the exact per-tile pipeline as [`write_tiles`]
+/// (noise → predictor → ZSTD). The Perlin field is statistically homogeneous,
+/// so a fixed seed and any interior tiles are representative.
+fn measure_ratio(config: &CogConfig, freq: f32) -> io::Result<f32> {
+ let noise = PerlinNoise::new(0);
+ let (w, h, ts) = (
+ config.raster.cog_width,
+ config.raster.cog_height,
+ config.tile_size,
+ );
+ let bpp = config.dtype.bytes_per_pixel();
+ // Clamp the sample block to the available grid (tiny configs / tests).
+ let nx = CALIB_TILES.min(w.div_ceil(ts));
+ let ny = CALIB_TILES.min(h.div_ceil(ts));
+ let mut tile_buf = vec![0u8; ts as usize * ts as usize * bpp];
+ let mut compressor = zstd::bulk::Compressor::new(config.zstd_level)
+ .map_err(|e| io::Error::other(format!("ZSTD compressor init failed: {e}")))?;
+ let (mut raw, mut comp) = (0usize, 0usize);
+ for ty in 0..ny {
+ for tx in 0..nx {
+ match config.dtype {
+ RasterDtype::UInt8 => {
+ noise.generate_tile_u8_into(w, h, freq, ts, tx, ty, &mut tile_buf)
+ }
+ RasterDtype::UInt16 => {
+ noise.generate_tile_u16_into(w, h, freq, ts, tx, ty, &mut tile_buf)
+ }
+ RasterDtype::Float32 => {
+ noise.generate_tile_f32_into(w, h, freq, ts, tx, ty, &mut tile_buf)
+ }
+ }
+ apply_predictor(&mut tile_buf, ts as usize, ts as usize, config.dtype);
+ let c = compressor
+ .compress(&tile_buf)
+ .map_err(|e| io::Error::other(format!("ZSTD compression failed: {e}")))?;
+ raw += tile_buf.len();
+ comp += c.len();
+ }
+ }
+ Ok(raw as f32 / comp.max(1) as f32)
+}
+
/// Compress and write all tiles, returning offset and byte count arrays.
///
/// Compression uses ZSTD (TIFF tag 50000) at `config.zstd_level` with a
@@ -757,6 +895,87 @@ mod tests {
}
}
+ fn calib_config(dtype: RasterDtype) -> CogConfig {
+ // 2048×2048 / tile 512 → a 4×4 block of full interior tiles; fast.
+ CogConfig {
+ // Real Sentinel-2-sized image: the descending (smooth-noise) arm
+ // spans the whole realistic ratio range here. measure_ratio samples
+ // a fixed 4×4 tile block regardless of dimensions, so this is fast.
+ raster: FootprintConfig {
+ cog_width: 10980,
+ cog_height: 10980,
+ resolution: 10,
+ },
+ tile_size: 512,
+ noise_frequency: 8.0, // ignored by calibration
+ dtype,
+ zstd_level: 3,
+ }
+ }
+
+ #[test]
+ fn calibrate_hits_target() {
+ let cal = calibrate_frequency(&calib_config(RasterDtype::UInt16), 2.0).unwrap();
+ assert!(!cal.clamped, "2.0x should be reachable at 10980²");
+ assert!(
+ (cal.achieved_ratio - 2.0).abs() / 2.0 < 0.10,
+ "achieved {} not within 10% of 2.0",
+ cal.achieved_ratio
+ );
+ assert!(
+ cal.frequency > CALIB_FREQ_MIN && cal.frequency < CALIB_FREQ_MAX,
+ "frequency {} should be interior",
+ cal.frequency
+ );
+ }
+
+ #[test]
+ fn calibrate_is_deterministic() {
+ let cfg = calib_config(RasterDtype::UInt16);
+ let a = calibrate_frequency(&cfg, 2.0).unwrap();
+ let b = calibrate_frequency(&cfg, 2.0).unwrap();
+ assert_eq!(a.frequency.to_bits(), b.frequency.to_bits());
+ }
+
+ #[test]
+ fn calibrate_clamps_unreachable_high() {
+ // No frequency makes smooth Perlin compress 100x → clamp to the arm
+ // start (lowest frequency, highest ratio).
+ let cal = calibrate_frequency(&calib_config(RasterDtype::UInt16), 100.0).unwrap();
+ assert!(cal.clamped);
+ assert_eq!(cal.frequency.to_bits(), CALIB_FREQ_MIN.to_bits());
+ }
+
+ #[test]
+ fn calibrate_clamps_unreachable_low() {
+ // ~1.02x is below the arm's minimum ratio → clamp to the arm end
+ // (highest frequency reached). Achieved ratio stays above the target.
+ let cal = calibrate_frequency(&calib_config(RasterDtype::UInt16), 1.02).unwrap();
+ assert!(cal.clamped);
+ assert!(cal.frequency > CALIB_FREQ_MIN, "should clamp to the high end");
+ assert!(cal.achieved_ratio > 1.02);
+ }
+
+ #[test]
+ fn calibrate_dtype_dependent() {
+ // Same target, different dtype → different resolved frequency. This is
+ // the footgun fix: a fixed noise_frequency is not portable across dtypes.
+ let u8_freq = calibrate_frequency(&calib_config(RasterDtype::UInt8), 2.0)
+ .unwrap()
+ .frequency;
+ let f32_freq = calibrate_frequency(&calib_config(RasterDtype::Float32), 2.0)
+ .unwrap()
+ .frequency;
+ assert_ne!(u8_freq.to_bits(), f32_freq.to_bits());
+ }
+
+ #[test]
+ fn measure_ratio_decreases_in_smooth_regime() {
+ // On the descending arm, more frequency = more detail = lower ratio.
+ let cfg = calib_config(RasterDtype::UInt16);
+ assert!(measure_ratio(&cfg, 16.0).unwrap() > measure_ratio(&cfg, 256.0).unwrap());
+ }
+
/// Test with dimensions that aren't a multiple of tile_size.
/// Default: 1830×1830 with 256 tiles → 8×8 tiles, last column/row
/// extends past the image (8×256 = 2048 > 1830).
@@ -918,3 +1137,4 @@ mod tests {
assert!(pixels.iter().all(|&v| (0.0..=1.0).contains(&v)));
}
}
+