You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pr...@apache.org on 2026/06/02 19:46:25 UTC

(sedona-spatialbench) 03/06: feat(raster): bounded streaming S3 upload pipeline with resume

This is an automated email from the ASF dual-hosted git repository.

prantogg pushed a commit to branch pranav/feature/raster-benchmark
in repository https://gitbox.apache.org/repos/asf/sedona-spatialbench.git

commit dbc13ae8619e18cd7c2c0a5c79dcf1071d0583e9
Author: Pranav Toggi <pr...@gmail.com>
AuthorDate: Mon Jun 1 22:35:40 2026 -0700

    feat(raster): bounded streaming S3 upload pipeline with resume
    
    The raster runner gated only generation with a semaphore and dropped the
    permit before upload, leaving uploads unbounded — at realistic ~125 MB
    COGs a large run accumulates buffers until OOM. Rewrite it as a bounded
    two-stage futures pipeline (CPU generate -> I/O write) with independent
    concurrency limits, so peak memory is a function of concurrency, not the
    total COG count. Unifies local and S3 output behind an OutputSink.
    
    Upload concurrency auto-derives from a memory budget
    (clamp(budget/raw_cog_bytes, gen, 4*gen)), overridable via
    --raster-upload-concurrency / --raster-memory-budget-mb.
    
    Adds --resume: skip COGs already present at the destination (S3 LIST /
    local walk). The manifest is built from the full work list independent of
    skips, so the STAC catalog always covers the whole pile.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
 spatialbench-cli/src/main.rs          |  32 +-
 spatialbench-cli/src/raster_runner.rs | 738 +++++++++++++++++++++++++++-------
 2 files changed, 628 insertions(+), 142 deletions(-)

diff --git a/spatialbench-cli/src/main.rs b/spatialbench-cli/src/main.rs
index 9e2f59e..db71d04 100644
--- a/spatialbench-cli/src/main.rs
+++ b/spatialbench-cli/src/main.rs
@@ -148,6 +148,23 @@ struct Cli {
     /// Maximum number of raster footprints to generate (limits output size for fast iteration).
     #[arg(long)]
     max_footprints: Option<u32>,
+
+    /// Skip raster COGs already present at the output destination (and skip
+    /// re-uploading them). Assumes existing COGs were generated with the same
+    /// configuration (frequency/dtype/dimensions) — changing config and
+    /// resuming will keep stale COGs. Default: off (overwrite).
+    #[arg(long, default_value_t = false)]
+    resume: bool,
+
+    /// Max concurrent raster output operations (S3 PUTs / file writes).
+    /// Defaults to an auto-derived value from --raster-memory-budget-mb.
+    #[arg(long)]
+    raster_upload_concurrency: Option<usize>,
+
+    /// Memory budget (MiB) for in-flight raster COG buffers; controls the
+    /// auto-derived upload concurrency. Default: 8192.
+    #[arg(long)]
+    raster_memory_budget_mb: Option<u64>,
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
@@ -428,13 +445,16 @@ impl Cli {
                     .to_string_lossy()
                     .into_owned()
             };
-            let manifest = raster_runner::run_raster(
-                &footprints,
+            let manifest = raster_runner::run_raster(raster_runner::RunRasterArgs {
+                footprints: &footprints,
                 tier,
-                &cog_config,
-                &raster_dir,
-                self.num_threads,
-            )
+                cog_config: &cog_config,
+                output_dir: &raster_dir,
+                num_threads: self.num_threads,
+                resume: self.resume,
+                upload_concurrency: self.raster_upload_concurrency,
+                memory_budget_bytes: self.raster_memory_budget_mb.map(|mb| mb * 1024 * 1024),
+            })
             .await?;
 
             info!("generated {} manifest entries", manifest.len());
diff --git a/spatialbench-cli/src/raster_runner.rs b/spatialbench-cli/src/raster_runner.rs
index 16c5a89..458e3e3 100644
--- a/spatialbench-cli/src/raster_runner.rs
+++ b/spatialbench-cli/src/raster_runner.rs
@@ -17,12 +17,17 @@
 
 //! Parallel raster COG generation pipeline.
 //!
-//! Uses a semaphore-bounded pool of `spawn_blocking` workers, each with a
-//! thread-local pixel buffer reused across COGs. Manifest entries flow back
-//! through a bounded `mpsc` channel.
+//! A bounded two-stage `futures` stream: a CPU stage generates COG bytes in
+//! `spawn_blocking` workers (each with a thread-local pixel buffer reused
+//! across COGs), and an I/O stage writes them to a local directory or uploads
+//! to S3. The two stages have independent concurrency limits, so peak memory
+//! is a function of concurrency — not the total number of COGs — making
+//! multi-TB runs memory-safe regardless of size.
 //!
-//! For S3 output, COGs are encoded to `Vec<u8>` in memory and uploaded via
-//! `object_store::put()`. For local output, bytes are written to files.
+//! With `resume`, COGs already present at the destination are skipped (both
+//! generation and upload). The returned manifest is always built from the full
+//! work list, so the STAC catalog covers the whole pile even on a resume that
+//! generates nothing.
 
 use crate::s3_writer::{build_s3_client, parse_s3_uri};
 
@@ -32,188 +37,649 @@ use spatialbench_raster::scaling::ScalingTier;
 use spatialbench_raster::ManifestEntry;
 
 use bytes::Bytes;
+use futures::stream::{StreamExt, TryStreamExt};
 use log::info;
 use object_store::path::Path as ObjectPath;
 use object_store::ObjectStore;
-use tokio::sync::{mpsc, Semaphore};
 
 use std::cell::RefCell;
+use std::collections::HashSet;
 use std::io;
 use std::path::Path;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
+/// Default memory budget for in-flight COG buffers when none is supplied: 8 GiB.
+const DEFAULT_RASTER_MEMORY_BUDGET_BYTES: u64 = 8 * 1024 * 1024 * 1024;
+
 /// A single unit of work: generate one COG.
 #[derive(Clone)]
 struct CogWorkItem {
     footprint: Footprint,
     cog_id: u32,
-    /// Relative path within the output directory (e.g., "pile/00000/0000.tif").
+    /// Pile-relative key, e.g. "00000/0000.tif".
     key: String,
     config: CogConfig,
 }
 
-/// Generate COGs for all footprints in parallel and return manifest entries.
+/// Destination for generated COG bytes. Keys are relative to the pile root,
+/// formatted as "{fp:05}/{cog:04}.tif" (matches [`CogWorkItem::key`]).
+enum OutputSink {
+    Local {
+        /// Path to the `pile` directory.
+        pile_dir: String,
+    },
+    S3 {
+        client: Arc<dyn ObjectStore>,
+        /// Object-key prefix up to (not including) the trailing slash,
+        /// e.g. "output/raster/pile".
+        prefix: String,
+    },
+    /// Instrumented sink for pipeline tests: counts puts, tracks concurrent
+    /// in-flight puts, and can inject latency or a failure.
+    #[cfg(test)]
+    Test {
+        inflight: Arc<std::sync::atomic::AtomicUsize>,
+        max_inflight: Arc<std::sync::atomic::AtomicUsize>,
+        put_count: Arc<AtomicU64>,
+        /// 1-based put index that should return an error (`None` = never).
+        fail_at: Option<u64>,
+        delay_ms: u64,
+    },
+}
+
+impl OutputSink {
+    /// Write one COG. `key` is the pile-relative key, e.g. "00007/0003.tif".
+    async fn put(&self, key: &str, bytes: Vec<u8>) -> io::Result<()> {
+        match self {
+            OutputSink::Local { pile_dir } => {
+                let path = format!("{pile_dir}/{key}");
+                tokio::fs::write(&path, &bytes).await
+            }
+            OutputSink::S3 { client, prefix } => {
+                let path = ObjectPath::from(format!("{prefix}/{key}"));
+                client
+                    .put(&path, Bytes::from(bytes).into())
+                    .await
+                    .map(|_| ())
+                    .map_err(|e| io::Error::other(format!("S3 PUT failed: {e}")))
+            }
+            #[cfg(test)]
+            OutputSink::Test {
+                inflight,
+                max_inflight,
+                put_count,
+                fail_at,
+                delay_ms,
+            } => {
+                let cur = inflight.fetch_add(1, Ordering::SeqCst) + 1;
+                max_inflight.fetch_max(cur, Ordering::SeqCst);
+                if *delay_ms > 0 {
+                    tokio::time::sleep(std::time::Duration::from_millis(*delay_ms)).await;
+                }
+                let n = put_count.fetch_add(1, Ordering::SeqCst) + 1;
+                inflight.fetch_sub(1, Ordering::SeqCst);
+                if Some(n) == *fail_at {
+                    return Err(io::Error::other("injected put failure"));
+                }
+                Ok(())
+            }
+        }
+    }
+
+    /// Return the set of pile-relative keys ("{fp:05}/{cog:04}.tif") that
+    /// already exist at this destination. Used only for `resume`.
+    async fn existing_keys(&self) -> io::Result<HashSet<String>> {
+        match self {
+            OutputSink::Local { pile_dir } => local_existing_keys(pile_dir),
+            OutputSink::S3 { client, prefix } => s3_existing_keys(client, prefix).await,
+            #[cfg(test)]
+            OutputSink::Test { .. } => Ok(HashSet::new()),
+        }
+    }
+}
+
+/// List existing pile objects in S3 with one paginated LIST and strip the
+/// prefix down to pile-relative `.tif` keys.
+async fn s3_existing_keys(
+    client: &Arc<dyn ObjectStore>,
+    prefix: &str,
+) -> io::Result<HashSet<String>> {
+    let list_prefix = ObjectPath::from(prefix);
+    let mut stream = client.list(Some(&list_prefix));
+    let strip = format!("{prefix}/");
+    let mut keys = HashSet::new();
+    while let Some(meta) = stream.next().await {
+        let meta = meta.map_err(|e| io::Error::other(format!("S3 LIST failed: {e}")))?;
+        if let Some(rel) = meta.location.as_ref().strip_prefix(&strip) {
+            if rel.ends_with(".tif") {
+                keys.insert(rel.to_string());
+            }
+        }
+    }
+    Ok(keys)
+}
+
+/// Walk a local pile directory (`pile/{fp}/{cog}.tif`) collecting existing
+/// pile-relative keys. A missing pile directory yields an empty set.
+fn local_existing_keys(pile_dir: &str) -> io::Result<HashSet<String>> {
+    let mut keys = HashSet::new();
+    let root = Path::new(pile_dir);
+    if !root.exists() {
+        return Ok(keys);
+    }
+    for fp_entry in std::fs::read_dir(root)? {
+        let fp_entry = fp_entry?;
+        if !fp_entry.file_type()?.is_dir() {
+            continue;
+        }
+        let fp_name = fp_entry.file_name();
+        let fp_name = fp_name.to_string_lossy();
+        for cog_entry in std::fs::read_dir(fp_entry.path())? {
+            let cog_entry = cog_entry?;
+            let cog_name = cog_entry.file_name();
+            let cog_name = cog_name.to_string_lossy();
+            if cog_name.ends_with(".tif") {
+                keys.insert(format!("{fp_name}/{cog_name}"));
+            }
+        }
+    }
+    Ok(keys)
+}
+
+/// Resolved concurrency limits for the two pipeline stages.
+#[derive(Debug, Clone, Copy)]
+struct RasterConcurrency {
+    /// CPU-bound generation stage.
+    gen: usize,
+    /// Network/memory-bound output stage.
+    upload: usize,
+}
+
+impl RasterConcurrency {
+    /// Derive limits.
+    ///
+    /// - `gen` = `num_threads`.
+    /// - `upload` = `clamp(memory_budget / est_cog_bytes, gen, 4 * gen)`,
+    ///   unless `upload_override` is `Some`, which wins (clamped to `>= 1`).
+    ///
+    /// `est_cog_bytes` is the conservative per-in-flight-COG memory unit (raw
+    /// pixel size); raw ≥ compressed output, so this keeps peak memory under
+    /// budget even for incompressible data.
+    fn resolve(
+        num_threads: usize,
+        est_cog_bytes: u64,
+        memory_budget_bytes: u64,
+        upload_override: Option<usize>,
+    ) -> Self {
+        let gen = num_threads.max(1);
+        let upload = match upload_override {
+            Some(u) => u.max(1),
+            None => {
+                let by_budget = (memory_budget_bytes / est_cog_bytes.max(1)) as usize;
+                by_budget.clamp(gen, gen * 4)
+            }
+        };
+        Self { gen, upload }
+    }
+}
+
+/// Raw bytes of one COG's pixel buffer — the dominant in-flight memory unit.
+fn est_cog_bytes(cog: &CogConfig) -> u64 {
+    cog.raster.cog_width as u64
+        * cog.raster.cog_height as u64
+        * cog.dtype.bytes_per_pixel() as u64
+}
+
+/// Build the full work list and the full deterministic manifest.
 ///
-/// Pipeline:
-/// 1. Pre-create all footprint directories (skipped for S3 output).
-/// 2. Spawn all work items eagerly; semaphore gates concurrency to `num_threads`.
-/// 3. Each `spawn_blocking` worker uses a `thread_local!` pixel buffer.
-/// 4. COG bytes are written to local files or uploaded to S3 via `object_store`.
-/// 5. Manifest entries flow through a bounded `mpsc` channel.
-/// 6. A collector task drains the channel into `Vec<ManifestEntry>`.
-pub async fn run_raster(
+/// The manifest is independent of which COGs are later skipped or generated,
+/// so the STAC catalog always describes the entire pile.
+fn build_work_and_manifest(
     footprints: &[Footprint],
     tier: &ScalingTier,
     cog_config: &CogConfig,
-    output_dir: &str,
-    num_threads: usize,
-) -> io::Result<Vec<ManifestEntry>> {
-    let is_s3 = output_dir.starts_with("s3://");
-    let total_cogs = footprints.len() as u64 * tier.scenes_per_footprint as u64;
-
-    // Build S3 client if needed, otherwise prepare local pile directory
-    let s3 = if is_s3 {
+) -> (Vec<CogWorkItem>, Vec<ManifestEntry>) {
+    let cap = footprints.len() * tier.scenes_per_footprint as usize;
+    let mut work_items = Vec::with_capacity(cap);
+    let mut manifest = Vec::with_capacity(cap);
+    for fp in footprints {
+        for cog_id in 0..tier.scenes_per_footprint {
+            work_items.push(CogWorkItem {
+                footprint: *fp,
+                cog_id,
+                key: format!("{:05}/{:04}.tif", fp.id, cog_id),
+                config: *cog_config,
+            });
+            manifest.push(ManifestEntry {
+                footprint_id: fp.id,
+                cog_id,
+                bbox_4326: fp.bbox_4326,
+                epsg: fp.epsg,
+            });
+        }
+    }
+    (work_items, manifest)
+}
+
+/// Drop work items whose key already exists at the destination.
+fn filter_pending(work: Vec<CogWorkItem>, existing: &HashSet<String>) -> Vec<CogWorkItem> {
+    work.into_iter()
+        .filter(|w| !existing.contains(&w.key))
+        .collect()
+}
+
+/// Generate one COG's bytes, reusing a thread-local pixel buffer.
+fn generate_cog(item: CogWorkItem) -> io::Result<(Vec<u8>, String)> {
+    thread_local! {
+        static PIXEL_BUF: RefCell<Option<PixelBuffer>> = const { RefCell::new(None) };
+    }
+    PIXEL_BUF.with(|cell| {
+        let mut opt = cell.borrow_mut();
+        let buf = opt.get_or_insert_with(|| PixelBuffer::new(item.config.dtype));
+        let bytes = write_cog_bytes(&item.config, &item.footprint, item.cog_id, buf)?;
+        Ok((bytes, item.key))
+    })
+}
+
+/// Run the bounded two-stage pipeline over `items`, writing to `sink`.
+///
+/// Stage 1 (CPU) generates at most `conc.gen` COGs concurrently; stage 2 (I/O)
+/// writes at most `conc.upload` concurrently. Because the stages are chained,
+/// the CPU stage cannot run more than ~`conc.gen` ahead of the I/O stage, so
+/// peak in-flight COGs ≈ `gen + upload` — independent of `items.len()`.
+async fn run_pipeline(
+    sink: Arc<OutputSink>,
+    items: Vec<CogWorkItem>,
+    conc: RasterConcurrency,
+    total_cogs: u64,
+) -> io::Result<()> {
+    let counter = Arc::new(AtomicU64::new(0));
+
+    futures::stream::iter(items.into_iter())
+        // Stage 1 (CPU): generate bytes off the async runtime.
+        .map(|item| async move {
+            tokio::task::spawn_blocking(move || generate_cog(item))
+                .await
+                .map_err(|e| io::Error::other(format!("blocking task panicked: {e}")))?
+        })
+        .buffer_unordered(conc.gen)
+        // Stage 2 (I/O): write/upload, bounded independently.
+        .map(|res: io::Result<(Vec<u8>, String)>| {
+            let sink = Arc::clone(&sink);
+            let counter = Arc::clone(&counter);
+            async move {
+                let (bytes, key) = res?;
+                sink.put(&key, bytes).await?;
+                let n = counter.fetch_add(1, Ordering::Relaxed) + 1;
+                if n.is_multiple_of(100) {
+                    info!("generated {n}/{total_cogs} COGs");
+                }
+                Ok::<(), io::Error>(())
+            }
+        })
+        .buffer_unordered(conc.upload)
+        .try_for_each(|()| async { Ok(()) })
+        .await
+}
+
+/// Parameters for [`run_raster`].
+pub struct RunRasterArgs<'a> {
+    pub footprints: &'a [Footprint],
+    pub tier: &'a ScalingTier,
+    pub cog_config: &'a CogConfig,
+    pub output_dir: &'a str,
+    pub num_threads: usize,
+    /// Skip COGs already present at the destination (LIST/walk on start).
+    pub resume: bool,
+    /// Explicit output-stage concurrency; `None` auto-derives from budget.
+    pub upload_concurrency: Option<usize>,
+    /// In-flight memory budget in bytes; `None` uses the 8 GiB default.
+    pub memory_budget_bytes: Option<u64>,
+}
+
+/// Generate COGs for all footprints and return manifest entries.
+///
+/// See the module docs for the pipeline shape. Output goes to a local `pile`
+/// directory or, for `s3://` URIs, to S3.
+pub async fn run_raster(args: RunRasterArgs<'_>) -> io::Result<Vec<ManifestEntry>> {
+    let RunRasterArgs {
+        footprints,
+        tier,
+        cog_config,
+        output_dir,
+        num_threads,
+        resume,
+        upload_concurrency,
+        memory_budget_bytes,
+    } = args;
+
+    // Build the output sink.
+    let sink = if output_dir.starts_with("s3://") {
         let (bucket, prefix) = parse_s3_uri(output_dir.trim_end_matches('/'))?;
         let client = build_s3_client(&bucket)?;
-        Some((client, format!("{}/pile", prefix)))
+        OutputSink::S3 {
+            client,
+            prefix: format!("{prefix}/pile"),
+        }
     } else {
-        None
+        let pile_dir = Path::new(output_dir)
+            .join("pile")
+            .to_string_lossy()
+            .into_owned();
+        OutputSink::Local { pile_dir }
     };
 
-    let local_pile_dir = if !is_s3 {
-        let p = Path::new(output_dir).join("pile");
-        Some(p.to_string_lossy().into_owned())
+    // Pre-create local footprint directories (S3 has no directories).
+    if let OutputSink::Local { ref pile_dir } = sink {
+        for fp in footprints {
+            std::fs::create_dir_all(format!("{pile_dir}/{:05}", fp.id))?;
+        }
+    }
+
+    let (work_items, manifest) = build_work_and_manifest(footprints, tier, cog_config);
+    let total_cogs = work_items.len() as u64;
+
+    // Resume: drop already-present COGs.
+    let pending = if resume {
+        let existing = sink.existing_keys().await?;
+        let before = work_items.len();
+        let pending = filter_pending(work_items, &existing);
+        info!(
+            "resume: {} of {} COGs already present, generating {}",
+            before - pending.len(),
+            before,
+            pending.len()
+        );
+        pending
     } else {
-        None
+        work_items
     };
 
-    // Phase 1: Pre-create directories (local only; S3 has no directories)
-    if let Some(ref pile_dir) = local_pile_dir {
-        for fp in footprints {
-            std::fs::create_dir_all(format!("{}/{:05}", pile_dir, fp.id))?;
+    let conc = RasterConcurrency::resolve(
+        num_threads,
+        est_cog_bytes(cog_config),
+        memory_budget_bytes.unwrap_or(DEFAULT_RASTER_MEMORY_BUDGET_BYTES),
+        upload_concurrency,
+    );
+    info!(
+        "raster pipeline: gen_concurrency={}, upload_concurrency={}, pending={}",
+        conc.gen,
+        conc.upload,
+        pending.len()
+    );
+
+    run_pipeline(Arc::new(sink), pending, conc, total_cogs).await?;
+
+    info!("generated {} COGs (complete)", total_cogs);
+    Ok(manifest)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use object_store::memory::InMemory;
+    use spatialbench_raster::cog::RasterDtype;
+    use spatialbench_raster::footprint::FootprintConfig;
+    use std::sync::atomic::AtomicUsize;
+
+    fn tiny_cog_config() -> CogConfig {
+        CogConfig {
+            raster: FootprintConfig {
+                cog_width: 64,
+                cog_height: 64,
+                resolution: 60,
+            },
+            tile_size: 32,
+            noise_frequency: 4.0,
+            dtype: RasterDtype::UInt8,
+            zstd_level: 1,
         }
     }
 
-    // Phase 2: Build work items
-    let mut work_items = Vec::with_capacity(total_cogs as usize);
-    for fp in footprints {
-        for cog_id in 0..tier.scenes_per_footprint {
-            work_items.push(CogWorkItem {
-                footprint: *fp,
-                cog_id,
-                key: format!("{:05}/{:04}.tif", fp.id, cog_id),
-                config: *cog_config,
-            });
+    fn footprint(id: u32) -> Footprint {
+        Footprint {
+            id,
+            epsg: 32614,
+            origin: (500_000.0, 4_000_000.0),
+            bbox_4326: [-100.0, 35.0, -99.0, 36.0],
         }
     }
 
-    // Phase 3: Parallel execution
-    let semaphore = Arc::new(Semaphore::new(num_threads));
-    let counter = Arc::new(AtomicU64::new(0));
-    let (tx, mut rx) = mpsc::channel::<ManifestEntry>(num_threads * 2);
-    // Collector task
-    let collector = tokio::spawn(async move {
-        let mut manifest = Vec::with_capacity(total_cogs as usize);
-        while let Some(entry) = rx.recv().await {
-            manifest.push(entry);
+    fn tier(scenes: u32) -> ScalingTier {
+        ScalingTier {
+            sf: 1,
+            scenes_per_footprint: scenes,
+            narrow: (1, scenes),
+            balanced: (1, scenes),
+            wide: (1, scenes),
         }
-        manifest
-    });
-
-    // Spawn work items with backpressure; semaphore acquired before spawn
-    // to prevent eagerly queuing millions of tasks in memory.
-    let mut join_handles = Vec::with_capacity(work_items.len());
-    for item in work_items {
-        // Acquire permit BEFORE spawning to apply backpressure to the loop
-        let permit = Arc::clone(&semaphore)
-            .acquire_owned()
-            .await
-            .map_err(|e| io::Error::other(format!("semaphore closed: {e}")))?;
-
-        let tx = tx.clone();
-        let counter = Arc::clone(&counter);
-        let total = total_cogs;
-        let s3 = s3.clone();
-        let local_pile_dir = local_pile_dir.clone();
-
-        let handle = tokio::spawn(async move {
-            let cog_output = tokio::task::spawn_blocking(move || {
-                thread_local! {
-                    static PIXEL_BUF: RefCell<Option<PixelBuffer>> = const { RefCell::new(None) };
-                }
+    }
+
+    // ---- RasterConcurrency::resolve ----
 
-                PIXEL_BUF.with(|cell| {
-                    let mut opt = cell.borrow_mut();
-                    let buf = opt.get_or_insert_with(|| PixelBuffer::new(item.config.dtype));
-                    let bytes = write_cog_bytes(&item.config, &item.footprint, item.cog_id, buf)?;
-
-                    Ok::<(Vec<u8>, String, ManifestEntry), io::Error>((
-                        bytes,
-                        item.key,
-                        ManifestEntry {
-                            footprint_id: item.footprint.id,
-                            cog_id: item.cog_id,
-                            bbox_4326: item.footprint.bbox_4326,
-                            epsg: item.footprint.epsg,
-                        },
-                    ))
-                })
-            })
+    #[test]
+    fn resolve_budget_caps_low() {
+        // 1 GiB / 241 MB ≈ 4, clamped up to gen.
+        let c = RasterConcurrency::resolve(12, 241_000_000, 1024 * 1024 * 1024, None);
+        assert_eq!(c.gen, 12);
+        assert_eq!(c.upload, 12);
+    }
+
+    #[test]
+    fn resolve_budget_mid() {
+        let c = RasterConcurrency::resolve(12, 241_000_000, 8 * 1024 * 1024 * 1024, None);
+        assert_eq!(c.upload, 35); // 8 GiB / 241 MB
+    }
+
+    #[test]
+    fn resolve_budget_huge_clamps_to_4x() {
+        let c = RasterConcurrency::resolve(12, 241_000_000, 1024u64.pow(4), None);
+        assert_eq!(c.upload, 48);
+    }
+
+    #[test]
+    fn resolve_override_wins() {
+        let c = RasterConcurrency::resolve(12, 241_000_000, 8 * 1024 * 1024 * 1024, Some(5));
+        assert_eq!(c.upload, 5);
+    }
+
+    #[test]
+    fn resolve_override_floor_is_one() {
+        let c = RasterConcurrency::resolve(12, 241_000_000, 8 * 1024 * 1024 * 1024, Some(0));
+        assert_eq!(c.upload, 1);
+    }
+
+    #[test]
+    fn resolve_est_zero_no_panic() {
+        let c = RasterConcurrency::resolve(12, 0, 8 * 1024 * 1024 * 1024, None);
+        assert_eq!(c.upload, 48);
+    }
+
+    // ---- OutputSink round-trips ----
+
+    #[tokio::test]
+    async fn s3_put_and_list_roundtrip() {
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let sink = OutputSink::S3 {
+            client: Arc::clone(&store),
+            prefix: "out/raster/pile".to_string(),
+        };
+        sink.put("00000/0000.tif", vec![1, 2, 3]).await.unwrap();
+        // A non-.tif object under the prefix must be excluded.
+        store
+            .put(
+                &ObjectPath::from("out/raster/pile/_marker.txt"),
+                Bytes::from_static(b"x").into(),
+            )
             .await
-            .map_err(|e| io::Error::other(format!("blocking task panicked: {e}")))??;
+            .unwrap();
 
-            // Release permit after generation, before I/O — lets other workers
-            // start generating while this task uploads.
-            drop(permit);
+        let keys = sink.existing_keys().await.unwrap();
+        assert_eq!(keys, HashSet::from(["00000/0000.tif".to_string()]));
+    }
 
-            let (bytes, key, entry) = cog_output;
+    #[tokio::test]
+    async fn local_put_and_walk_roundtrip() {
+        let dir = tempfile::tempdir().unwrap();
+        let pile_dir = dir.path().join("pile").to_string_lossy().into_owned();
+        std::fs::create_dir_all(format!("{pile_dir}/00000")).unwrap();
+        std::fs::create_dir_all(format!("{pile_dir}/00001")).unwrap();
+        let sink = OutputSink::Local {
+            pile_dir: pile_dir.clone(),
+        };
+        sink.put("00000/0000.tif", vec![1]).await.unwrap();
+        sink.put("00001/0003.tif", vec![2]).await.unwrap();
+        // Stray non-.tif file must be excluded.
+        std::fs::write(format!("{pile_dir}/00000/notes.txt"), b"x").unwrap();
 
-            // Write output: S3 upload or local file
-            if let Some((ref client, ref prefix)) = s3 {
-                let path = ObjectPath::from(format!("{}/{}", prefix, key));
-                let payload = Bytes::from(bytes);
-                client
-                    .put(&path, payload.into())
-                    .await
-                    .map_err(|e| io::Error::other(format!("S3 PUT failed: {e}")))?;
-            } else if let Some(ref pile_dir) = local_pile_dir {
-                let path = format!("{}/{}", pile_dir, key);
-                tokio::fs::write(&path, &bytes).await?;
-            }
+        let keys = sink.existing_keys().await.unwrap();
+        assert_eq!(
+            keys,
+            HashSet::from(["00000/0000.tif".to_string(), "00001/0003.tif".to_string()])
+        );
+    }
 
-            let n = counter.fetch_add(1, Ordering::Relaxed) + 1;
-            if n.is_multiple_of(100) {
-                info!("generated {n}/{total} COGs");
+    #[tokio::test]
+    async fn existing_keys_empty_destination() {
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let s3 = OutputSink::S3 {
+            client: store,
+            prefix: "out/pile".to_string(),
+        };
+        assert!(s3.existing_keys().await.unwrap().is_empty());
+
+        let dir = tempfile::tempdir().unwrap();
+        let local = OutputSink::Local {
+            pile_dir: dir.path().join("pile").to_string_lossy().into_owned(),
+        };
+        assert!(local.existing_keys().await.unwrap().is_empty());
+    }
+
+    // ---- work list + manifest + resume filter ----
+
+    #[test]
+    fn work_and_manifest_cover_all_pairs() {
+        let fps = [footprint(0), footprint(1)];
+        let (work, manifest) = build_work_and_manifest(&fps, &tier(3), &tiny_cog_config());
+        assert_eq!(work.len(), 6);
+        assert_eq!(manifest.len(), 6);
+        let keys: HashSet<_> = work.iter().map(|w| w.key.clone()).collect();
+        for fp in 0..2 {
+            for cog in 0..3 {
+                assert!(keys.contains(&format!("{fp:05}/{cog:04}.tif")));
             }
+        }
+    }
 
-            tx.send(entry)
-                .await
-                .map_err(|e| io::Error::other(format!("channel send failed: {e}")))?;
+    #[test]
+    fn filter_pending_drops_existing_only() {
+        let fps = [footprint(0)];
+        let (work, _) = build_work_and_manifest(&fps, &tier(4), &tiny_cog_config());
+        let existing = HashSet::from(["00000/0000.tif".to_string(), "00000/0002.tif".to_string()]);
+        let pending = filter_pending(work, &existing);
+        assert_eq!(pending.len(), 2);
+        assert!(pending.iter().all(|w| !existing.contains(&w.key)));
+    }
 
-            Ok::<(), io::Error>(())
-        });
-        join_handles.push(handle);
+    // ---- run_raster end-to-end (local) ----
+
+    #[tokio::test]
+    async fn run_raster_local_generates_then_resumes() {
+        let dir = tempfile::tempdir().unwrap();
+        let out = dir.path().to_string_lossy().into_owned();
+        let fps = [footprint(0), footprint(1)];
+        let tier = tier(2);
+        let cfg = tiny_cog_config();
+
+        let args = |resume| RunRasterArgs {
+            footprints: &fps,
+            tier: &tier,
+            cog_config: &cfg,
+            output_dir: &out,
+            num_threads: 2,
+            resume,
+            upload_concurrency: None,
+            memory_budget_bytes: None,
+        };
+
+        let manifest = run_raster(args(false)).await.unwrap();
+        assert_eq!(manifest.len(), 4);
+        let pile = dir.path().join("pile");
+        let count_tifs = || {
+            let mut n = 0;
+            for fp in std::fs::read_dir(&pile).unwrap() {
+                for cog in std::fs::read_dir(fp.unwrap().path()).unwrap() {
+                    if cog.unwrap().file_name().to_string_lossy().ends_with(".tif") {
+                        n += 1;
+                    }
+                }
+            }
+            n
+        };
+        assert_eq!(count_tifs(), 4);
+
+        // Resume: manifest still full, files still present (skipped).
+        let manifest2 = run_raster(args(true)).await.unwrap();
+        assert_eq!(manifest2.len(), 4);
+        assert_eq!(count_tifs(), 4);
     }
 
-    // Wait for all workers, propagating first error
-    for handle in join_handles {
-        handle
-            .await
-            .map_err(|e| io::Error::other(format!("task join failed: {e}")))??;
+    // ---- run_pipeline: backpressure + error propagation ----
+
+    fn pipeline_items(n: u32) -> Vec<CogWorkItem> {
+        let cfg = tiny_cog_config();
+        let (work, _) = build_work_and_manifest(&[footprint(0)], &tier(n), &cfg);
+        work
     }
 
-    // Drop sender so collector finishes
-    drop(tx);
-    let manifest = collector
-        .await
-        .map_err(|e| io::Error::other(format!("collector task failed: {e}")))?;
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn pipeline_upload_stage_is_bounded() {
+        // Slow uploads + many items: the upload stage must never exceed its
+        // configured concurrency, regardless of how many items exist. This is
+        // the memory-safety invariant (output buffers live in the upload stage).
+        let inflight = Arc::new(AtomicUsize::new(0));
+        let max_inflight = Arc::new(AtomicUsize::new(0));
+        let put_count = Arc::new(AtomicU64::new(0));
+        let sink = Arc::new(OutputSink::Test {
+            inflight: Arc::clone(&inflight),
+            max_inflight: Arc::clone(&max_inflight),
+            put_count: Arc::clone(&put_count),
+            fail_at: None,
+            delay_ms: 5,
+        });
+        let conc = RasterConcurrency {
+            gen: 4,
+            upload: 3,
+        };
+        let items = pipeline_items(60);
+        run_pipeline(sink, items, conc, 60).await.unwrap();
 
-    info!(
-        "generated {}/{} COGs (complete)",
-        counter.load(Ordering::Relaxed),
-        total_cogs
-    );
+        assert_eq!(put_count.load(Ordering::SeqCst), 60);
+        assert!(
+            max_inflight.load(Ordering::SeqCst) <= conc.upload,
+            "upload in-flight {} exceeded limit {}",
+            max_inflight.load(Ordering::SeqCst),
+            conc.upload
+        );
+    }
 
-    Ok(manifest)
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn pipeline_propagates_put_error() {
+        let sink = Arc::new(OutputSink::Test {
+            inflight: Arc::new(AtomicUsize::new(0)),
+            max_inflight: Arc::new(AtomicUsize::new(0)),
+            put_count: Arc::new(AtomicU64::new(0)),
+            fail_at: Some(3),
+            delay_ms: 0,
+        });
+        let conc = RasterConcurrency {
+            gen: 2,
+            upload: 2,
+        };
+        let err = run_pipeline(sink, pipeline_items(20), conc, 20)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("injected put failure"));
+    }
 }