You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/08/18 17:25:55 UTC

(iggy) branch master updated: feat(connectors): add source batch acknowledgments (#3855)

This is an automated email from the ASF dual-hosted git repository.

hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git


The following commit(s) were added to refs/heads/master by this push:
     new af9ce9548 feat(connectors): add source batch acknowledgments (#3855)
af9ce9548 is described below

commit af9ce954841a3b0c54ed4a59e2a1db714fe84a5a
Author: Rohan Dubey <ro...@gmail.com>
AuthorDate: Tue Aug 18 22:55:50 2026 +0530

    feat(connectors): add source batch acknowledgments (#3855)
---
 core/connectors/runtime/src/main.rs                |  14 +-
 core/connectors/runtime/src/manager/source.rs      |  28 +-
 core/connectors/runtime/src/source.rs              | 364 ++++++++-
 core/connectors/sdk/README.md                      |  29 +
 core/connectors/sdk/src/lib.rs                     |  12 +
 core/connectors/sdk/src/source.rs                  | 897 ++++++++++++++++++++-
 core/connectors/sources/README.md                  |  33 +-
 core/connectors/sources/random_source/src/lib.rs   |  91 ++-
 .../tests/connectors/random/random_source.rs       | 123 +++
 9 files changed, 1518 insertions(+), 73 deletions(-)

diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs
index 5c5ebe777..54c6b466a 100644
--- a/core/connectors/runtime/src/main.rs
+++ b/core/connectors/runtime/src/main.rs
@@ -29,7 +29,7 @@ use iggy_connector_sdk::{
     StreamDecoder, StreamEncoder,
     api::ConnectorStatus,
     sink::ConsumeCallback,
-    source::{HandleCallback, SendCallback},
+    source::{BatchResultCallback, HandleCallback, SendCallback},
     transforms::Transform,
 };
 use mimalloc::MiMalloc;
@@ -80,7 +80,8 @@ pub(crate) struct SourceApi {
         state_len: usize,
         log_callback: iggy_connector_sdk::LogCallback,
     ) -> i32,
-    iggy_source_handle: extern "C" fn(id: u32, callback: SendCallback) -> i32,
+    iggy_source_handle_v2: extern "C" fn(id: u32, callback: SendCallback) -> i32,
+    iggy_source_batch_result: extern "C" fn(plugin_id: u32, batch_id: u64, result: u8) -> i32,
     iggy_source_close: extern "C" fn(id: u32) -> i32,
     iggy_source_version: extern "C" fn() -> *const std::ffi::c_char,
 }
@@ -185,12 +186,14 @@ async fn main() -> Result<(), RuntimeError> {
     let mut source_containers_by_key: HashMap<String, Arc<Container<SourceApi>>> = HashMap::new();
     for (_path, source) in sources {
         let container = Arc::new(source.container);
-        let callback = container.iggy_source_handle;
+        let handle_callback = container.iggy_source_handle_v2;
+        let batch_result_callback = container.iggy_source_batch_result;
         for plugin in &source.plugins {
             source_containers_by_key.insert(plugin.key.clone(), container.clone());
         }
         source_wrappers.push(SourceConnectorWrapper {
-            callback,
+            handle_callback,
+            batch_result_callback,
             plugins: source.plugins,
         });
     }
@@ -460,7 +463,8 @@ struct SourceConnectorProducer {
 }
 
 struct SourceConnectorWrapper {
-    callback: HandleCallback,
+    handle_callback: HandleCallback,
+    batch_result_callback: BatchResultCallback,
     plugins: Vec<SourceConnectorPlugin>,
 }
 
diff --git a/core/connectors/runtime/src/manager/source.rs b/core/connectors/runtime/src/manager/source.rs
index b011472fe..674b06e23 100644
--- a/core/connectors/runtime/src/manager/source.rs
+++ b/core/connectors/runtime/src/manager/source.rs
@@ -109,6 +109,17 @@ impl SourceManager {
         }
     }
 
+    pub async fn is_stopping_or_stopped(&self, key: &str) -> bool {
+        let Some(source) = self.sources.get(key).map(|entry| entry.value().clone()) else {
+            return true;
+        };
+        let source = source.lock().await;
+        matches!(
+            source.info.status,
+            ConnectorStatus::Stopping | ConnectorStatus::Stopped
+        )
+    }
+
     pub async fn stop_connector_with_guard(
         &self,
         key: &str,
@@ -138,6 +149,9 @@ impl SourceManager {
             .map(|e| e.value().clone())
             .ok_or_else(|| RuntimeError::SourceNotFound(key.to_string()))?;
 
+        self.update_status(key, ConnectorStatus::Stopping, Some(metrics))
+            .await;
+
         let (task_handles, plugin_id, container) = {
             let mut details = details.lock().await;
             (
@@ -147,8 +161,8 @@ impl SourceManager {
             )
         };
 
-        // Order: close FFI (stops callbacks) -> drop sender (unblocks
-        // recv_async) -> await tasks. Reversing risks an abort mid-save.
+        // The result callback can race with close, so the forwarding loop
+        // treats callback failures during Stopping as expected shutdown.
         if let Some(container) = &container {
             info!("Closing source connector with ID: {plugin_id} for plugin: {key}");
             (container.iggy_source_close)(plugin_id);
@@ -172,12 +186,8 @@ impl SourceManager {
 
         {
             let mut details = details.lock().await;
-            let old_status = details.info.status;
             details.info.status = ConnectorStatus::Stopped;
             details.info.last_error = None;
-            if old_status == ConnectorStatus::Running {
-                metrics.decrement_sources_running();
-            }
         }
 
         Ok(())
@@ -223,7 +233,8 @@ impl SourceManager {
         let (producer, encoder, transforms) =
             source::setup_source_producer(key, config, iggy_client).await?;
 
-        let callback = container.iggy_source_handle;
+        let handle_callback = container.iggy_source_handle_v2;
+        let batch_result_callback = container.iggy_source_batch_result;
         let handler_tasks = source::spawn_source_handler(
             plugin_id,
             key,
@@ -233,7 +244,8 @@ impl SourceManager {
             encoder,
             transforms,
             state_storage,
-            callback,
+            handle_callback,
+            batch_result_callback,
             context.clone(),
         );
 
diff --git a/core/connectors/runtime/src/source.rs b/core/connectors/runtime/src/source.rs
index e259e212c..d63496f0b 100644
--- a/core/connectors/runtime/src/source.rs
+++ b/core/connectors/runtime/src/source.rs
@@ -25,10 +25,12 @@ use iggy::prelude::{
 use iggy_connector_sdk::encoders::avro::{AvroEncoderConfig, AvroStreamEncoder};
 use iggy_connector_sdk::{
     ConnectorState, DecodedMessage, ProducedMessages, Schema, StreamEncoder, TopicMetadata,
-    source::HandleCallback, transforms::Transform,
+    source::{BatchResultCallback, HandleCallback, SourceBatchResult},
+    transforms::Transform,
 };
 use std::{
     collections::{BTreeMap, HashMap},
+    future::Future,
     str::FromStr,
     sync::{Arc, LazyLock, atomic::Ordering},
     time::Instant,
@@ -50,13 +52,21 @@ use iggy_connector_sdk::api::ConnectorStatus;
 use prometheus_client::metrics::counter::Counter;
 use tokio::task::JoinHandle;
 
+const MAX_FAILED_TAIL_RETRIES: u32 = 3;
+
 pub(crate) struct SourceSenderEntry {
-    pub(crate) sender: Sender<ProducedMessages>,
+    pub(crate) sender: Sender<ProducedBatch>,
     // Owned errors counter (Arc<AtomicU64> inside) so the FFI callback bumps
     // it with one relaxed atomic - no Family RwLock + HashMap lookup per call.
     pub(crate) error_counter: Counter,
 }
 
+#[derive(Debug)]
+pub(crate) struct ProducedBatch {
+    id: u64,
+    messages: ProducedMessages,
+}
+
 pub(crate) static SOURCE_SENDERS: LazyLock<DashMap<u32, SourceSenderEntry>> =
     LazyLock::new(DashMap::new);
 
@@ -364,7 +374,8 @@ pub(crate) async fn source_forwarding_loop(
     encoder: Arc<dyn StreamEncoder>,
     transforms: Vec<Arc<dyn Transform>>,
     state_storage: StateStorage,
-    receiver: Receiver<ProducedMessages>,
+    receiver: Receiver<ProducedBatch>,
+    batch_result_callback: BatchResultCallback,
     context: Arc<RuntimeContext>,
     labels: Arc<SourceLabels>,
 ) {
@@ -390,8 +401,10 @@ pub(crate) async fn source_forwarding_loop(
         topic: producer.topic().to_string(),
     };
 
-    while let Ok(produced_messages) = receiver.recv_async().await {
+    while let Ok(produced_batch) = receiver.recv_async().await {
         let total_start = Instant::now();
+        let batch_id = produced_batch.id;
+        let produced_messages = produced_batch.messages;
         let count = produced_messages.messages.len();
         context
             .metrics
@@ -437,7 +450,7 @@ pub(crate) async fn source_forwarding_loop(
             .observe_stage_with_labels(&labels.stage_decode, decode_elapsed);
 
         let prepare_start = Instant::now();
-        let iggy_messages = process_messages(
+        let processed = process_messages(
             plugin_id,
             &encoder,
             &topic_metadata,
@@ -450,10 +463,23 @@ pub(crate) async fn source_forwarding_loop(
         context
             .metrics
             .observe_stage_with_labels(&labels.stage_prepare, prepare_elapsed);
-        let sent_count = iggy_messages.len();
+        let prepared_count = processed.messages.len();
+        let processing_errors = decode_errors + processed.error_count;
 
         let iggy_send_start = Instant::now();
-        let send_result = producer.send(iggy_messages).await;
+        let send_result = if processing_errors == 0 {
+            send_with_failed_tail_retries(processed.messages, plugin_id, |messages| {
+                producer.send(messages)
+            })
+            .await
+        } else {
+            Err(IggyError::Error)
+        };
+        let sent_count = if send_result.is_ok() {
+            prepared_count
+        } else {
+            0
+        };
         let iggy_send_elapsed = iggy_send_start.elapsed();
         context
             .metrics
@@ -461,12 +487,19 @@ pub(crate) async fn source_forwarding_loop(
 
         // Total histogram + emit (below) run regardless of send outcome.
         let mut state_save_us: Option<u64> = None;
+        let mut batch_result = SourceBatchResult::Nack;
         if let Err(error) = send_result {
-            let error_msg = format!(
-                "Failed to send {sent_count} messages to stream: {}, topic: {} by source connector with ID: {plugin_id}. {error}",
-                producer.stream(),
-                producer.topic(),
-            );
+            let error_msg = if processing_errors > 0 {
+                format!(
+                    "Rejected source batch {batch_id} after {processing_errors} decode or processing errors for source connector with ID: {plugin_id}"
+                )
+            } else {
+                format!(
+                    "Failed to send {prepared_count} messages to stream: {}, topic: {} by source connector with ID: {plugin_id}. {error}",
+                    producer.stream(),
+                    producer.topic(),
+                )
+            };
             error!("{error_msg}");
             context.metrics.inc_errors_with_labels(&labels.counter);
             context.sources.set_error(&plugin_key, &error_msg).await;
@@ -489,11 +522,13 @@ pub(crate) async fn source_forwarding_loop(
                 );
             }
 
+            let mut state_saved = true;
             if let Some(state) = produced_messages.state {
                 let state_save_start = Instant::now();
                 match &state_storage {
                     StateStorage::File(file) => {
                         if let Err(error) = file.save(state).await {
+                            state_saved = false;
                             let error_msg = format!(
                                 "Failed to save state for source connector with ID: {plugin_id}. {error}"
                             );
@@ -514,6 +549,31 @@ pub(crate) async fn source_forwarding_loop(
             } else {
                 debug!("No state provided for source connector with ID: {plugin_id}");
             }
+
+            if state_saved {
+                batch_result = SourceBatchResult::Ack;
+            }
+        }
+
+        // The plugin applies its async batch-result hook before this FFI call returns.
+        let result_code = tokio::task::spawn_blocking(move || {
+            batch_result_callback(plugin_id, batch_id, batch_result as u8)
+        })
+        .await
+        .unwrap_or(-1);
+        if result_code != 0 {
+            if context.sources.is_stopping_or_stopped(&plugin_key).await {
+                trace!(
+                    "Source connector with ID: {plugin_id} stopped before {batch_result:?} could be delivered for batch ID: {batch_id}"
+                );
+            } else {
+                let error_msg = format!(
+                    "Failed to deliver {batch_result:?} for source connector with ID: {plugin_id}, batch ID: {batch_id}. Plugin returned: {result_code}"
+                );
+                error!("{error_msg}");
+                context.metrics.inc_errors_with_labels(&labels.counter);
+                context.sources.set_error(&plugin_key, &error_msg).await;
+            }
         }
 
         let total_elapsed = total_start.elapsed();
@@ -558,7 +618,8 @@ pub(crate) fn spawn_source_handler(
     encoder: Arc<dyn StreamEncoder>,
     transforms: Vec<Arc<dyn Transform>>,
     state_storage: StateStorage,
-    callback: HandleCallback,
+    handle_callback: HandleCallback,
+    batch_result_callback: BatchResultCallback,
     context: Arc<RuntimeContext>,
 ) -> Vec<JoinHandle<()>> {
     let (sender, receiver) = flume::unbounded();
@@ -573,7 +634,7 @@ pub(crate) fn spawn_source_handler(
     );
 
     let blocking_handle = tokio::task::spawn_blocking(move || {
-        callback(plugin_id, handle_produced_messages);
+        handle_callback(plugin_id, handle_produced_messages);
     });
     let handler_task = tokio::spawn(async move {
         source_forwarding_loop(
@@ -586,6 +647,7 @@ pub(crate) fn spawn_source_handler(
             transforms,
             state_storage,
             receiver,
+            batch_result_callback,
             context,
             labels,
         )
@@ -627,7 +689,8 @@ pub fn handle(
                 producer_wrapper.encoder,
                 plugin.transforms,
                 plugin.state_storage,
-                source.callback,
+                source.handle_callback,
+                source.batch_result_callback,
                 context.clone(),
             );
 
@@ -637,6 +700,11 @@ pub fn handle(
     handles
 }
 
+struct ProcessedMessages {
+    messages: Vec<IggyMessage>,
+    error_count: u64,
+}
+
 fn process_messages(
     id: u32,
     encoder: &Arc<dyn StreamEncoder>,
@@ -645,7 +713,7 @@ fn process_messages(
     transforms: &Vec<Arc<dyn Transform>>,
     metrics: &Arc<crate::metrics::Metrics>,
     labels: &SourceLabels,
-) -> Vec<IggyMessage> {
+) -> ProcessedMessages {
     let mut iggy_messages = Vec::with_capacity(messages.len());
     // Accumulate per-message drops, flush once after the loop - one Family
     // lookup instead of one per message under filter/error storms.
@@ -708,14 +776,72 @@ fn process_messages(
     if filtered_count > 0 {
         metrics.inc_messages_filtered_with_labels(&labels.counter, filtered_count);
     }
-    iggy_messages
+    ProcessedMessages {
+        messages: iggy_messages,
+        error_count,
+    }
+}
+
+async fn send_with_failed_tail_retries<F, Fut, T>(
+    mut messages: Vec<IggyMessage>,
+    plugin_id: u32,
+    mut send: F,
+) -> Result<(), IggyError>
+where
+    F: FnMut(Vec<IggyMessage>) -> Fut,
+    Fut: Future<Output = Result<T, IggyError>>,
+{
+    let mut retry = 0;
+    loop {
+        match send(messages).await {
+            Ok(_) => return Ok(()),
+            Err(IggyError::ProducerSendFailed {
+                cause,
+                failed,
+                committed,
+                stream_name,
+                topic_name,
+            }) => {
+                warn!(
+                    "Source connector with ID: {plugin_id} send failed after {} chunks committed; {} messages remain",
+                    committed.len(),
+                    failed.len()
+                );
+                if retry >= MAX_FAILED_TAIL_RETRIES || failed.is_empty() {
+                    return Err(IggyError::ProducerSendFailed {
+                        cause,
+                        failed,
+                        committed,
+                        stream_name,
+                        topic_name,
+                    });
+                }
+
+                messages = match Arc::try_unwrap(failed) {
+                    Ok(messages) => messages,
+                    Err(failed) => {
+                        return Err(IggyError::ProducerSendFailed {
+                            cause,
+                            failed,
+                            committed,
+                            stream_name,
+                            topic_name,
+                        });
+                    }
+                };
+                retry += 1;
+            }
+            Err(error) => return Err(error),
+        }
+    }
 }
 
 pub(crate) extern "C" fn handle_produced_messages(
     plugin_id: u32,
+    batch_id: u64,
     messages_ptr: *const u8,
     messages_len: usize,
-) {
+) -> i32 {
     unsafe {
         // Entry missing = SOURCE_SENDERS cleaned up at shutdown; benign race
         // expected on stop/restart. No metric (would conflate with real failures).
@@ -724,23 +850,29 @@ pub(crate) extern "C" fn handle_produced_messages(
                 plugin_id,
                 "dropping produced batch: sender already cleaned up"
             );
-            return;
+            return -1;
         };
         let messages = std::slice::from_raw_parts(messages_ptr, messages_len);
         match postcard::from_bytes::<ProducedMessages>(messages) {
             Ok(messages) => {
-                if let Err(send_error) = entry.sender.send(messages) {
+                if let Err(send_error) = entry.sender.send(ProducedBatch {
+                    id: batch_id,
+                    messages,
+                }) {
                     error!(
                         "Failed to send messages for source connector with ID: {plugin_id}. Channel closed: {send_error}"
                     );
                     entry.error_counter.inc();
+                    return -1;
                 }
+                0
             }
             Err(err) => {
                 error!(
                     "Failed to deserialize produced messages for source connector with ID: {plugin_id}. {err}"
                 );
                 entry.error_counter.inc();
+                -1
             }
         }
     }
@@ -768,3 +900,195 @@ fn build_iggy_message(
         (None, None) => IggyMessage::builder().payload(payload.into()).build(),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::VecDeque;
+    use std::future::ready;
+    use std::sync::atomic::{AtomicU32, Ordering};
+
+    static TEST_PLUGIN_ID: AtomicU32 = AtomicU32::new(u32::MAX / 2);
+
+    fn next_plugin_id() -> u32 {
+        TEST_PLUGIN_ID.fetch_add(1, Ordering::Relaxed)
+    }
+
+    fn test_message(marker: u8) -> IggyMessage {
+        IggyMessage::builder()
+            .payload(vec![marker].into())
+            .build()
+            .expect("test message should be valid")
+    }
+
+    fn failed_send(messages: Vec<IggyMessage>) -> IggyError {
+        IggyError::ProducerSendFailed {
+            cause: Box::new(IggyError::Error),
+            failed: Arc::new(messages),
+            committed: Arc::new(Vec::new()),
+            stream_name: "test-stream".to_string(),
+            topic_name: "test-topic".to_string(),
+        }
+    }
+
+    #[test]
+    fn given_serialized_batch_when_callback_runs_should_forward_batch_id() {
+        let plugin_id = next_plugin_id();
+        let batch_id = 73;
+        let (sender, receiver) = flume::unbounded();
+        SOURCE_SENDERS.insert(
+            plugin_id,
+            SourceSenderEntry {
+                sender,
+                error_counter: Counter::default(),
+            },
+        );
+        let messages = ProducedMessages {
+            schema: Schema::Raw,
+            messages: Vec::new(),
+            state: Some(ConnectorState(vec![1, 2, 3])),
+        };
+        let serialized = postcard::to_allocvec(&messages).expect("failed to serialize batch");
+
+        assert_eq!(
+            handle_produced_messages(plugin_id, batch_id, serialized.as_ptr(), serialized.len()),
+            0
+        );
+        let forwarded = receiver.recv().expect("batch was not forwarded");
+        assert_eq!(forwarded.id, batch_id);
+        assert_eq!(
+            forwarded
+                .messages
+                .state
+                .expect("state should be preserved")
+                .0,
+            vec![1, 2, 3]
+        );
+
+        cleanup_sender(plugin_id);
+    }
+
+    #[test]
+    fn given_invalid_payload_when_callback_runs_should_reject_batch() {
+        let plugin_id = next_plugin_id();
+        let (sender, _receiver) = flume::unbounded();
+        let error_counter = Counter::default();
+        SOURCE_SENDERS.insert(
+            plugin_id,
+            SourceSenderEntry {
+                sender,
+                error_counter: error_counter.clone(),
+            },
+        );
+        let invalid_payload = [0xff];
+
+        assert_eq!(
+            handle_produced_messages(
+                plugin_id,
+                1,
+                invalid_payload.as_ptr(),
+                invalid_payload.len(),
+            ),
+            -1
+        );
+        assert_eq!(error_counter.get(), 1);
+
+        cleanup_sender(plugin_id);
+    }
+
+    #[test]
+    fn given_missing_sender_when_callback_runs_should_reject_batch() {
+        let plugin_id = next_plugin_id();
+        let serialized = postcard::to_allocvec(&ProducedMessages {
+            schema: Schema::Raw,
+            messages: Vec::new(),
+            state: None,
+        })
+        .expect("failed to serialize batch");
+
+        assert_eq!(
+            handle_produced_messages(plugin_id, 1, serialized.as_ptr(), serialized.len()),
+            -1
+        );
+    }
+
+    #[test]
+    fn given_partially_committed_send_should_retry_only_failed_tail() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let mut attempts = Vec::new();
+            let mut responses = VecDeque::from([
+                Err(failed_send(vec![test_message(2), test_message(3)])),
+                Ok(()),
+            ]);
+
+            let result = send_with_failed_tail_retries(
+                vec![test_message(1), test_message(2), test_message(3)],
+                31,
+                |messages| {
+                    attempts.push(
+                        messages
+                            .iter()
+                            .map(|message| message.payload[0])
+                            .collect::<Vec<_>>(),
+                    );
+                    ready(responses.pop_front().expect("send response should exist"))
+                },
+            )
+            .await;
+
+            assert!(result.is_ok());
+            assert_eq!(attempts, vec![vec![1, 2, 3], vec![2, 3]]);
+        });
+    }
+
+    #[test]
+    fn given_repeated_failed_tail_when_retries_are_exhausted_should_return_error() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let mut attempts = 0;
+            let mut responses = VecDeque::from(
+                (0..=MAX_FAILED_TAIL_RETRIES)
+                    .map(|_| Err::<(), _>(failed_send(vec![test_message(1)])))
+                    .collect::<Vec<_>>(),
+            );
+
+            let result = send_with_failed_tail_retries(vec![test_message(1)], 37, |_| {
+                attempts += 1;
+                ready(responses.pop_front().expect("send response should exist"))
+            })
+            .await;
+
+            assert!(matches!(result, Err(IggyError::ProducerSendFailed { .. })));
+            assert_eq!(attempts, MAX_FAILED_TAIL_RETRIES + 1);
+        });
+    }
+
+    #[test]
+    fn given_shared_failed_tail_when_retrying_should_return_original_error() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let failed = Arc::new(vec![test_message(2)]);
+            let retained = Arc::clone(&failed);
+            let mut response = Some(Err::<(), _>(IggyError::ProducerSendFailed {
+                cause: Box::new(IggyError::Error),
+                failed,
+                committed: Arc::new(Vec::new()),
+                stream_name: "test-stream".to_string(),
+                topic_name: "test-topic".to_string(),
+            }));
+            let mut attempts = 0;
+
+            let result =
+                send_with_failed_tail_retries(vec![test_message(1), test_message(2)], 41, |_| {
+                    attempts += 1;
+                    ready(response.take().expect("send response should exist"))
+                })
+                .await;
+
+            assert!(matches!(result, Err(IggyError::ProducerSendFailed { .. })));
+            assert_eq!(attempts, 1);
+            assert_eq!(retained.len(), 1);
+        });
+    }
+}
diff --git a/core/connectors/sdk/README.md b/core/connectors/sdk/README.md
index fb85d055b..1c72e143d 100644
--- a/core/connectors/sdk/README.md
+++ b/core/connectors/sdk/README.md
@@ -4,6 +4,35 @@ SDK provides the commonly used structs and traits such as `Sink` and `Source`, a
 
 The macros automatically export the connector's version (from `CARGO_PKG_VERSION`) via FFI, allowing the runtime to report per-connector version information in the `/stats` endpoint.
 
+## Source delivery acknowledgment
+
+Source connectors use a one-in-flight-batch contract between the plugin and the runtime:
+
+1. `Source::poll()` returns messages and candidate state without committing cursor changes or destructive operations.
+2. The runtime sends the batch to Iggy and waits for the producer result.
+3. After a successful send, the runtime persists the candidate state.
+4. The runtime reports `SourceBatchResult::Ack` to the plugin. A send or state-save failure reports `SourceBatchResult::Nack` instead.
+5. `Source::on_batch_result()` commits or discards the plugin's staged work before the next poll starts.
+
+An empty batch follows the same handshake. A source should return `state: None` when an empty poll made no progress; this avoids an unnecessary state write and cannot persist state left over from a failed batch. Producer errors, including request timeouts, report a NACK. A successful send from the legacy Iggy server is still an ACK even though that server returns an empty confirmation list.
+
+The crash behavior is intentionally at-least-once:
+
+| Crash point | Recovery behavior |
+| --- | --- |
+| Before Iggy commits the batch | Persisted state is unchanged and the source can poll the batch again. |
+| After Iggy commits but before the runtime observes success | Persisted state is unchanged, so the batch may be delivered again. |
+| After send success but before state persistence | Persisted state is unchanged, so the batch may be delivered again. |
+| After state persistence but before the plugin processes the ACK | The restored state records the delivered batch. Deferred source-side cleanup may still be pending. |
+| After the plugin processes the ACK | The state and plugin cursor both record the delivered batch. |
+| After an in-memory confirmation and source cleanup, but before Iggy fsyncs | A server crash can lose the batch after source cleanup unless server-side `enforce_fsync` is enabled. |
+
+An ACK means that Iggy confirmed the batch in memory; durability depends on the server's `enforce_fsync` setting. Source-side ACK work should be idempotent because process termination can interrupt it. NACK handling must discard staged cursor changes and staged delete or mark operations so polling can redeliver the batch. The SDK retries NACKed batches with capped exponential backoff and stops after repeated NACKs.
+
+The default `Source::on_batch_result()` implementation is a no-op for sources without staged work. Sources that advance cursors, delete rows, or mark rows must override it. The SDK stops polling if the handler returns an error, preventing a failed rollback from advancing to another batch.
+
+This contract is a breaking FFI change. Source plugins must be rebuilt with the matching SDK. The runtime loads `iggy_source_handle_v2`, which supplies a batch ID to the runtime callback, and source plugins export `iggy_source_batch_result` for the corresponding ACK or NACK.
+
 Moreover, it contains both, the `decoders` and `encoders` modules, implementing either `StreamDecoder` or `StreamEncoder` traits, which are used when consuming or producing data from/to Iggy streams.
 
 SDK is WiP, and it'd certainly benefit from having the support of multiple format schemas, such as Protobuf, Avro, Flatbuffers etc. including decoding/encoding the data between the different formats (when applicable) and supporting the data transformations whenever possible (easy for JSON, but complex for Bincode for example).
diff --git a/core/connectors/sdk/src/lib.rs b/core/connectors/sdk/src/lib.rs
index c8ed2ff94..294fae8b6 100644
--- a/core/connectors/sdk/src/lib.rs
+++ b/core/connectors/sdk/src/lib.rs
@@ -111,6 +111,18 @@ pub trait Source: Send + Sync {
     /// Invoked every time a batch of messages is produced to the configured stream and topic.
     async fn poll(&self) -> Result<ProducedMessages, Error>;
 
+    /// Invoked after the runtime has finished processing the most recently polled batch.
+    ///
+    /// Sources that track cursors or perform destructive operations should stage those changes
+    /// in [`Source::poll`] and apply them only after receiving [`source::SourceBatchResult::Ack`].
+    /// A [`source::SourceBatchResult::Nack`] means the staged changes must be discarded so the
+    /// batch can be polled again. The SDK allows only one batch to be in flight at a time and
+    /// stops polling if this method returns an error. The default no-op is suitable only for
+    /// sources that have no staged cursor changes or destructive work.
+    async fn on_batch_result(&self, _result: source::SourceBatchResult) -> Result<(), Error> {
+        Ok(())
+    }
+
     /// Invoked when the source is closed, allowing it to perform any necessary cleanup.
     async fn close(&mut self) -> Result<(), Error>;
 }
diff --git a/core/connectors/sdk/src/source.rs b/core/connectors/sdk/src/source.rs
index 713e00c6a..532d2643f 100644
--- a/core/connectors/sdk/src/source.rs
+++ b/core/connectors/sdk/src/source.rs
@@ -16,11 +16,19 @@
 // under the License.
 
 use crate::log::{CallbackLayer, LogCallback};
-use crate::{ConnectorState, Error, Source, get_runtime};
+use crate::retry::exponential_backoff;
+use crate::{ConnectorState, Source, get_runtime};
 use serde::de::DeserializeOwned;
-use std::sync::Arc;
-use tokio::{sync::watch, task::JoinHandle};
-use tracing::{error, info};
+use std::sync::{
+    Arc, Mutex, MutexGuard, PoisonError,
+    atomic::{AtomicU32, Ordering},
+};
+use std::time::Duration;
+use tokio::{
+    sync::{oneshot, watch},
+    task::JoinHandle,
+};
+use tracing::{error, info, warn};
 use tracing_subscriber::{EnvFilter, Registry, layer::SubscriberExt, util::SubscriberInitExt};
 
 #[repr(C)]
@@ -34,7 +42,72 @@ pub struct RawMessage {
 
 pub type HandleCallback = extern "C" fn(plugin_id: u32, callback: SendCallback) -> i32;
 
-pub type SendCallback = extern "C" fn(plugin_id: u32, messages_ptr: *const u8, messages_len: usize);
+pub type SendCallback = extern "C" fn(
+    plugin_id: u32,
+    batch_id: u64,
+    messages_ptr: *const u8,
+    messages_len: usize,
+) -> i32;
+
+pub type BatchResultCallback = extern "C" fn(plugin_id: u32, batch_id: u64, result: u8) -> i32;
+
+const BATCH_RESULT_TIMEOUT: Duration = Duration::from_secs(30);
+const NACK_RETRY_DELAY: Duration = Duration::from_millis(100);
+const MAX_NACK_RETRY_DELAY: Duration = Duration::from_secs(5);
+const MAX_CONSECUTIVE_NACKS: u32 = 5;
+
+/// Delivery result for the single batch currently in flight from a source plugin.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum SourceBatchResult {
+    /// The runtime sent the complete batch and persisted its candidate state.
+    Ack = 0,
+    /// The runtime could not send the batch or persist its candidate state.
+    Nack = 1,
+}
+
+impl TryFrom<u8> for SourceBatchResult {
+    type Error = ();
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            0 => Ok(Self::Ack),
+            1 => Ok(Self::Nack),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Debug)]
+struct PendingBatch {
+    id: u64,
+    result_sender: oneshot::Sender<BatchCompletion>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BatchCompletion {
+    Applied(SourceBatchResult),
+    Stop,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BatchPolicy {
+    result_timeout: Duration,
+    nack_retry_delay: Duration,
+    max_nack_retry_delay: Duration,
+    max_consecutive_nacks: u32,
+}
+
+impl Default for BatchPolicy {
+    fn default() -> Self {
+        Self {
+            result_timeout: BATCH_RESULT_TIMEOUT,
+            nack_retry_delay: NACK_RETRY_DELAY,
+            max_nack_retry_delay: MAX_NACK_RETRY_DELAY,
+            max_consecutive_nacks: MAX_CONSECUTIVE_NACKS,
+        }
+    }
+}
 
 #[derive(Debug)]
 pub struct SourceContainer<T: Source + std::fmt::Debug> {
@@ -42,15 +115,19 @@ pub struct SourceContainer<T: Source + std::fmt::Debug> {
     source: Option<Arc<T>>,
     shutdown: Option<watch::Sender<()>>,
     task: Option<JoinHandle<()>>,
+    pending_batch: Arc<Mutex<Option<PendingBatch>>>,
+    consecutive_nacks: Arc<AtomicU32>,
 }
 
 impl<T: Source + std::fmt::Debug + 'static> SourceContainer<T> {
-    pub const fn new(id: u32) -> Self {
+    pub fn new(id: u32) -> Self {
         Self {
             id,
             source: None,
             shutdown: None,
             task: None,
+            pending_batch: Arc::new(Mutex::new(None)),
+            consecutive_nacks: Arc::new(AtomicU32::new(0)),
         }
     }
 
@@ -157,22 +234,60 @@ impl<T: Source + std::fmt::Debug + 'static> SourceContainer<T> {
         let (shutdown_tx, shutdown_rx) = watch::channel(());
         let plugin_id = self.id;
         let source = Arc::clone(source);
+        let pending_batch = Arc::clone(&self.pending_batch);
+        let consecutive_nacks = Arc::clone(&self.consecutive_nacks);
         let handle = runtime.spawn(async move {
-            let _ = handle_messages(plugin_id, source, callback, shutdown_rx).await;
+            handle_messages(
+                plugin_id,
+                source,
+                move |plugin_id, batch_id, messages_ptr, messages_len| {
+                    callback(plugin_id, batch_id, messages_ptr, messages_len)
+                },
+                shutdown_rx,
+                pending_batch,
+                consecutive_nacks,
+                BatchPolicy::default(),
+            )
+            .await;
         });
 
         self.shutdown = Some(shutdown_tx);
         self.task = Some(handle);
         0
     }
+
+    #[doc(hidden)]
+    pub fn complete_batch(&self, batch_id: u64, result: u8) -> i32 {
+        let Some(source) = self.source.as_ref() else {
+            error!("Source connector with ID: {} is not initialized.", self.id);
+            return -1;
+        };
+
+        complete_pending_batch(
+            &self.pending_batch,
+            source,
+            &self.consecutive_nacks,
+            batch_id,
+            result,
+            self.id,
+            MAX_CONSECUTIVE_NACKS,
+        )
+    }
 }
 
-async fn handle_messages<T: Source>(
+async fn handle_messages<T, F>(
     plugin_id: u32,
     source: Arc<T>,
-    callback: SendCallback,
+    callback: F,
     mut shutdown: watch::Receiver<()>,
-) -> Result<(), Error> {
+    pending_batch: Arc<Mutex<Option<PendingBatch>>>,
+    consecutive_nacks: Arc<AtomicU32>,
+    policy: BatchPolicy,
+) where
+    T: Source,
+    F: Fn(u32, u64, *const u8, usize) -> i32,
+{
+    let mut batch_id = 1u64;
     loop {
         tokio::select! {
             _ = shutdown.changed() => {
@@ -192,16 +307,247 @@ async fn handle_messages<T: Source>(
                     Ok(messages) => messages,
                     Err(err) => {
                         error!("Failed to serialize messages for source connector with ID: {plugin_id}. {err}");
+                        if matches!(
+                            apply_batch_result(
+                                &source,
+                                &consecutive_nacks,
+                                SourceBatchResult::Nack,
+                                plugin_id,
+                                policy.max_consecutive_nacks,
+                            )
+                            .await,
+                            BatchCompletion::Stop
+                        ) {
+                            break;
+                        }
+                        sleep_after_nack(&consecutive_nacks, policy).await;
                         continue;
                     }
                 };
 
-                callback(plugin_id, messages.as_ptr(), messages.len());
+                let (result_sender, result_receiver) = oneshot::channel();
+                {
+                    let mut pending = lock_pending_batch(&pending_batch);
+                    *pending = Some(PendingBatch {
+                        id: batch_id,
+                        result_sender,
+                    });
+                }
+
+                let callback_result = callback(plugin_id, batch_id, messages.as_ptr(), messages.len());
+                drop(messages);
+
+                if callback_result != 0 {
+                    if !clear_pending_batch(&pending_batch, batch_id, plugin_id) {
+                        break;
+                    }
+                    let completion = apply_batch_result(
+                        &source,
+                        &consecutive_nacks,
+                        SourceBatchResult::Nack,
+                        plugin_id,
+                        policy.max_consecutive_nacks,
+                    )
+                    .await;
+                    if matches!(completion, BatchCompletion::Stop) {
+                        break;
+                    }
+                    sleep_after_nack(&consecutive_nacks, policy).await;
+                    batch_id += 1;
+                    continue;
+                }
+
+                let (completion, shutting_down) = tokio::select! {
+                    biased;
+                    result = result_receiver => {
+                        (result.unwrap_or(BatchCompletion::Stop), false)
+                    },
+                    _ = shutdown.changed() => {
+                        let completion = if clear_pending_batch(&pending_batch, batch_id, plugin_id) {
+                            apply_batch_result(
+                                &source,
+                                &consecutive_nacks,
+                                SourceBatchResult::Nack,
+                                plugin_id,
+                                policy.max_consecutive_nacks,
+                            ).await
+                        } else {
+                            BatchCompletion::Stop
+                        };
+                        (completion, true)
+                    },
+                    _ = tokio::time::sleep(policy.result_timeout) => {
+                        warn!(
+                            "Timed out waiting for batch result for source connector with ID: {plugin_id}, batch ID: {batch_id}"
+                        );
+                        let completion = if clear_pending_batch(&pending_batch, batch_id, plugin_id) {
+                            apply_batch_result(
+                                &source,
+                                &consecutive_nacks,
+                                SourceBatchResult::Nack,
+                                plugin_id,
+                                policy.max_consecutive_nacks,
+                            ).await
+                        } else {
+                            BatchCompletion::Stop
+                        };
+                        (completion, false)
+                    }
+                };
+
+                if matches!(completion, BatchCompletion::Stop) {
+                    break;
+                }
+
+                if shutting_down {
+                    info!("Shutting down source connector with ID: {plugin_id}");
+                    break;
+                }
+
+                if matches!(
+                    completion,
+                    BatchCompletion::Applied(SourceBatchResult::Nack)
+                ) {
+                    sleep_after_nack(&consecutive_nacks, policy).await;
+                }
+                batch_id += 1;
             }
         }
     }
+}
 
-    Ok(())
+fn complete_pending_batch<T>(
+    pending_batch: &Mutex<Option<PendingBatch>>,
+    source: &Arc<T>,
+    consecutive_nacks: &Arc<AtomicU32>,
+    batch_id: u64,
+    result_code: u8,
+    plugin_id: u32,
+    max_consecutive_nacks: u32,
+) -> i32
+where
+    T: Source + 'static,
+{
+    let (result, invalid_result) = match SourceBatchResult::try_from(result_code) {
+        Ok(result) => (result, false),
+        Err(()) => {
+            error!(
+                "Invalid batch result: {result_code} for source connector with ID: {plugin_id}; treating it as Nack"
+            );
+            (SourceBatchResult::Nack, true)
+        }
+    };
+
+    let Some(current) = take_pending_batch(pending_batch, batch_id, plugin_id) else {
+        return -1;
+    };
+
+    let completion = get_runtime().block_on(apply_batch_result(
+        source,
+        consecutive_nacks,
+        result,
+        plugin_id,
+        max_consecutive_nacks,
+    ));
+    if current.result_sender.send(completion).is_err() {
+        error!(
+            "Failed to deliver batch result for source connector with ID: {plugin_id}, batch ID: {batch_id}"
+        );
+        return -1;
+    }
+
+    if invalid_result || matches!(completion, BatchCompletion::Stop) {
+        -1
+    } else {
+        0
+    }
+}
+
+fn take_pending_batch(
+    pending_batch: &Mutex<Option<PendingBatch>>,
+    batch_id: u64,
+    plugin_id: u32,
+) -> Option<PendingBatch> {
+    let mut pending = lock_pending_batch(pending_batch);
+    let Some(current) = pending.as_ref() else {
+        error!("No batch is awaiting a result for source connector with ID: {plugin_id}");
+        return None;
+    };
+    if current.id != batch_id {
+        error!(
+            "Batch result ID mismatch for source connector with ID: {plugin_id}. Expected: {}, received: {batch_id}",
+            current.id
+        );
+        return None;
+    }
+
+    pending.take_if(|current| current.id == batch_id)
+}
+
+fn clear_pending_batch(
+    pending_batch: &Mutex<Option<PendingBatch>>,
+    batch_id: u64,
+    plugin_id: u32,
+) -> bool {
+    let mut pending = lock_pending_batch(pending_batch);
+    if let Some(current) = pending.as_ref()
+        && current.id != batch_id
+    {
+        error!(
+            "Batch result ID mismatch for source connector with ID: {plugin_id}. Expected: {}, received: {batch_id}",
+            current.id
+        );
+        return false;
+    }
+    pending.take_if(|current| current.id == batch_id).is_some()
+}
+
+fn lock_pending_batch(
+    pending_batch: &Mutex<Option<PendingBatch>>,
+) -> MutexGuard<'_, Option<PendingBatch>> {
+    pending_batch.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+async fn apply_batch_result<T: Source>(
+    source: &Arc<T>,
+    consecutive_nacks: &AtomicU32,
+    result: SourceBatchResult,
+    plugin_id: u32,
+    max_consecutive_nacks: u32,
+) -> BatchCompletion {
+    if let Err(err) = source.on_batch_result(result).await {
+        error!("Failed to process {result:?} for source connector with ID: {plugin_id}. {err}");
+        return BatchCompletion::Stop;
+    }
+
+    let consecutive_nacks = match result {
+        SourceBatchResult::Ack => {
+            consecutive_nacks.store(0, Ordering::Relaxed);
+            0
+        }
+        SourceBatchResult::Nack => consecutive_nacks.fetch_add(1, Ordering::Relaxed) + 1,
+    };
+    if consecutive_nacks >= max_consecutive_nacks {
+        error!(
+            "Stopping source connector with ID: {plugin_id} after {consecutive_nacks} consecutive NACKs"
+        );
+        return BatchCompletion::Stop;
+    }
+
+    BatchCompletion::Applied(result)
+}
+
+async fn sleep_after_nack(consecutive_nacks: &AtomicU32, policy: BatchPolicy) {
+    tokio::time::sleep(nack_retry_delay(consecutive_nacks, policy)).await;
+}
+
+fn nack_retry_delay(consecutive_nacks: &AtomicU32, policy: BatchPolicy) -> Duration {
+    let attempt = consecutive_nacks.load(Ordering::Relaxed).saturating_sub(1);
+    exponential_backoff(
+        policy.nack_retry_delay,
+        attempt,
+        policy.max_nack_retry_delay,
+    )
 }
 
 #[macro_export]
@@ -254,7 +600,7 @@ macro_rules! source_connector {
 
         #[cfg(not(test))]
         #[unsafe(no_mangle)]
-        unsafe extern "C" fn iggy_source_handle(id: u32, callback: SendCallback) -> i32 {
+        unsafe extern "C" fn iggy_source_handle_v2(id: u32, callback: SendCallback) -> i32 {
             let Some(mut instance) = INSTANCES.get_mut(&id) else {
                 tracing::error!(
                     "Source connector with ID: {id} was not found and cannot be handled."
@@ -264,16 +610,31 @@ macro_rules! source_connector {
             instance.handle(callback)
         }
 
+        #[cfg(not(test))]
+        #[unsafe(no_mangle)]
+        extern "C" fn iggy_source_batch_result(id: u32, batch_id: u64, result: u8) -> i32 {
+            let Some(instance) = INSTANCES.get(&id) else {
+                tracing::error!(
+                    "Source connector with ID: {id} was not found and cannot complete batch {batch_id}."
+                );
+                return -1;
+            };
+            instance.complete_batch(batch_id, result)
+        }
+
         #[cfg(not(test))]
         #[unsafe(no_mangle)]
         unsafe extern "C" fn iggy_source_close(id: u32) -> i32 {
-            let Some(mut instance) = INSTANCES.remove(&id) else {
+            let Some(mut instance) = INSTANCES.get_mut(&id) else {
                 tracing::error!(
                     "Source connector with ID: {id} was not found and cannot be closed."
                 );
                 return -1;
             };
-            instance.1.close()
+            let result = instance.close();
+            drop(instance);
+            INSTANCES.remove(&id);
+            result
         }
 
         #[cfg(not(test))]
@@ -284,3 +645,509 @@ macro_rules! source_connector {
         }
     };
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{ProducedMessages, Schema};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::time::Duration;
+    use tokio::sync::mpsc;
+
+    fn test_policy() -> BatchPolicy {
+        BatchPolicy {
+            result_timeout: Duration::from_millis(500),
+            nack_retry_delay: Duration::from_millis(1),
+            max_nack_retry_delay: Duration::from_millis(2),
+            max_consecutive_nacks: MAX_CONSECUTIVE_NACKS,
+        }
+    }
+
+    #[derive(Debug, Default)]
+    struct TestSource {
+        polls: AtomicUsize,
+        results: Mutex<Vec<SourceBatchResult>>,
+        fail_batch_result: AtomicBool,
+    }
+
+    #[async_trait::async_trait]
+    impl Source for TestSource {
+        async fn open(&mut self) -> Result<(), crate::Error> {
+            Ok(())
+        }
+
+        async fn poll(&self) -> Result<ProducedMessages, crate::Error> {
+            self.polls.fetch_add(1, Ordering::SeqCst);
+            Ok(ProducedMessages {
+                schema: Schema::Raw,
+                messages: Vec::new(),
+                state: None,
+            })
+        }
+
+        async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), crate::Error> {
+            self.results
+                .lock()
+                .unwrap_or_else(PoisonError::into_inner)
+                .push(result);
+            if self.fail_batch_result.load(Ordering::SeqCst) {
+                return Err(crate::Error::Storage(
+                    "failed to apply batch result".to_string(),
+                ));
+            }
+            Ok(())
+        }
+
+        async fn close(&mut self) -> Result<(), crate::Error> {
+            Ok(())
+        }
+    }
+
+    async fn complete_test_batch(
+        pending_batch: Arc<Mutex<Option<PendingBatch>>>,
+        source: Arc<TestSource>,
+        consecutive_nacks: Arc<AtomicU32>,
+        batch_id: u64,
+        result: SourceBatchResult,
+        plugin_id: u32,
+    ) -> i32 {
+        tokio::task::spawn_blocking(move || {
+            complete_pending_batch(
+                &pending_batch,
+                &source,
+                &consecutive_nacks,
+                batch_id,
+                result as u8,
+                plugin_id,
+                MAX_CONSECUTIVE_NACKS,
+            )
+        })
+        .await
+        .expect("batch result task should complete")
+    }
+
+    #[test]
+    fn given_batch_without_result_should_not_poll_again() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource::default());
+            let pending_batch = Arc::new(Mutex::new(None));
+            let pending_for_task = Arc::clone(&pending_batch);
+            let consecutive_nacks = Arc::new(AtomicU32::new(0));
+            let nacks_for_task = Arc::clone(&consecutive_nacks);
+            let (shutdown_sender, shutdown_receiver) = watch::channel(());
+            let (batch_sender, mut batch_receiver) = mpsc::unbounded_channel();
+
+            let source_for_task = Arc::clone(&source);
+            let task = tokio::spawn(handle_messages(
+                7,
+                source_for_task,
+                move |_, batch_id, _, _| {
+                    batch_sender
+                        .send(batch_id)
+                        .expect("batch receiver should remain open");
+                    0
+                },
+                shutdown_receiver,
+                pending_for_task,
+                nacks_for_task,
+                test_policy(),
+            ));
+
+            let batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("first batch was not sent")
+                .expect("batch channel closed");
+            assert_eq!(batch_id, 1);
+            assert_eq!(source.polls.load(Ordering::SeqCst), 1);
+            assert!(
+                tokio::time::timeout(Duration::from_millis(50), batch_receiver.recv())
+                    .await
+                    .is_err(),
+                "source polled again before the first batch was completed"
+            );
+
+            assert_eq!(
+                complete_test_batch(
+                    Arc::clone(&pending_batch),
+                    Arc::clone(&source),
+                    Arc::clone(&consecutive_nacks),
+                    batch_id,
+                    SourceBatchResult::Ack,
+                    7,
+                )
+                .await,
+                0
+            );
+            let next_batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("source did not poll after ACK")
+                .expect("batch channel closed");
+            assert_eq!(next_batch_id, 2);
+
+            shutdown_sender
+                .send(())
+                .expect("source task should remain active");
+            task.await.expect("source task failed");
+            assert_eq!(
+                *source
+                    .results
+                    .lock()
+                    .unwrap_or_else(PoisonError::into_inner),
+                vec![SourceBatchResult::Ack, SourceBatchResult::Nack]
+            );
+        });
+    }
+
+    #[test]
+    fn given_nack_when_batch_is_pending_should_allow_redelivery() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource::default());
+            let pending_batch = Arc::new(Mutex::new(None));
+            let pending_for_task = Arc::clone(&pending_batch);
+            let consecutive_nacks = Arc::new(AtomicU32::new(0));
+            let nacks_for_task = Arc::clone(&consecutive_nacks);
+            let (shutdown_sender, shutdown_receiver) = watch::channel(());
+            let (batch_sender, mut batch_receiver) = mpsc::unbounded_channel();
+
+            let source_for_task = Arc::clone(&source);
+            let task = tokio::spawn(handle_messages(
+                9,
+                source_for_task,
+                move |_, batch_id, _, _| {
+                    batch_sender
+                        .send(batch_id)
+                        .expect("batch receiver should remain open");
+                    0
+                },
+                shutdown_receiver,
+                pending_for_task,
+                nacks_for_task,
+                test_policy(),
+            ));
+
+            let batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("first batch was not sent")
+                .expect("batch channel closed");
+            assert_eq!(
+                complete_test_batch(
+                    Arc::clone(&pending_batch),
+                    Arc::clone(&source),
+                    Arc::clone(&consecutive_nacks),
+                    batch_id,
+                    SourceBatchResult::Nack,
+                    9,
+                )
+                .await,
+                0
+            );
+            let next_batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("source did not poll after NACK")
+                .expect("batch channel closed");
+            assert_eq!(next_batch_id, 2);
+
+            shutdown_sender
+                .send(())
+                .expect("source task should remain active");
+            task.await.expect("source task failed");
+            assert_eq!(
+                *source
+                    .results
+                    .lock()
+                    .unwrap_or_else(PoisonError::into_inner),
+                vec![SourceBatchResult::Nack, SourceBatchResult::Nack]
+            );
+        });
+    }
+
+    #[test]
+    fn given_mismatched_batch_id_should_reject_result() {
+        let source = Arc::new(TestSource::default());
+        let consecutive_nacks = Arc::new(AtomicU32::new(0));
+        let pending_batch = Mutex::new(None);
+        let (result_sender, result_receiver) = oneshot::channel();
+        *lock_pending_batch(&pending_batch) = Some(PendingBatch {
+            id: 41,
+            result_sender,
+        });
+
+        assert_eq!(
+            complete_pending_batch(
+                &pending_batch,
+                &source,
+                &consecutive_nacks,
+                42,
+                SourceBatchResult::Ack as u8,
+                11,
+                MAX_CONSECUTIVE_NACKS,
+            ),
+            -1
+        );
+        assert_eq!(
+            complete_pending_batch(
+                &pending_batch,
+                &source,
+                &consecutive_nacks,
+                41,
+                SourceBatchResult::Ack as u8,
+                11,
+                MAX_CONSECUTIVE_NACKS,
+            ),
+            0
+        );
+
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        assert_eq!(
+            runtime
+                .block_on(result_receiver)
+                .expect("batch result sender was dropped"),
+            BatchCompletion::Applied(SourceBatchResult::Ack)
+        );
+    }
+
+    #[test]
+    fn given_unknown_result_code_should_complete_batch_as_nack() {
+        let source = Arc::new(TestSource::default());
+        let consecutive_nacks = Arc::new(AtomicU32::new(0));
+        let pending_batch = Mutex::new(None);
+        let (result_sender, result_receiver) = oneshot::channel();
+        *lock_pending_batch(&pending_batch) = Some(PendingBatch {
+            id: 51,
+            result_sender,
+        });
+
+        assert_eq!(
+            complete_pending_batch(
+                &pending_batch,
+                &source,
+                &consecutive_nacks,
+                51,
+                99,
+                17,
+                MAX_CONSECUTIVE_NACKS,
+            ),
+            -1
+        );
+
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        assert_eq!(
+            runtime
+                .block_on(result_receiver)
+                .expect("batch result sender was dropped"),
+            BatchCompletion::Applied(SourceBatchResult::Nack)
+        );
+        assert_eq!(
+            *source
+                .results
+                .lock()
+                .unwrap_or_else(PoisonError::into_inner),
+            vec![SourceBatchResult::Nack]
+        );
+    }
+
+    #[test]
+    fn given_result_timeout_should_nack_and_poll_next_batch() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource::default());
+            let pending_batch = Arc::new(Mutex::new(None));
+            let consecutive_nacks = Arc::new(AtomicU32::new(0));
+            let (_shutdown_sender, shutdown_receiver) = watch::channel(());
+            let (batch_sender, mut batch_receiver) = mpsc::unbounded_channel();
+            let policy = BatchPolicy {
+                result_timeout: Duration::from_millis(10),
+                ..test_policy()
+            };
+
+            let task = tokio::spawn(handle_messages(
+                19,
+                Arc::clone(&source),
+                move |_, batch_id, _, _| {
+                    batch_sender
+                        .send(batch_id)
+                        .expect("batch receiver should remain open");
+                    0
+                },
+                shutdown_receiver,
+                Arc::clone(&pending_batch),
+                Arc::clone(&consecutive_nacks),
+                policy,
+            ));
+
+            assert_eq!(batch_receiver.recv().await, Some(1));
+            let next_batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("source did not poll after timed-out batch")
+                .expect("batch channel closed");
+            assert_eq!(next_batch_id, 2);
+            assert_eq!(
+                *source
+                    .results
+                    .lock()
+                    .unwrap_or_else(PoisonError::into_inner),
+                vec![SourceBatchResult::Nack]
+            );
+
+            task.abort();
+            let _ = task.await;
+        });
+    }
+
+    #[test]
+    fn given_batch_result_handler_failure_should_stop_polling() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource {
+                fail_batch_result: AtomicBool::new(true),
+                ..TestSource::default()
+            });
+            let pending_batch = Arc::new(Mutex::new(None));
+            let pending_for_task = Arc::clone(&pending_batch);
+            let consecutive_nacks = Arc::new(AtomicU32::new(0));
+            let nacks_for_task = Arc::clone(&consecutive_nacks);
+            let (_shutdown_sender, shutdown_receiver) = watch::channel(());
+            let (batch_sender, mut batch_receiver) = mpsc::unbounded_channel();
+
+            let source_for_task = Arc::clone(&source);
+            let task = tokio::spawn(handle_messages(
+                13,
+                source_for_task,
+                move |_, batch_id, _, _| {
+                    batch_sender
+                        .send(batch_id)
+                        .expect("batch receiver should remain open");
+                    0
+                },
+                shutdown_receiver,
+                pending_for_task,
+                nacks_for_task,
+                test_policy(),
+            ));
+
+            let batch_id = tokio::time::timeout(Duration::from_secs(1), batch_receiver.recv())
+                .await
+                .expect("first batch was not sent")
+                .expect("batch channel closed");
+            assert_eq!(
+                complete_test_batch(
+                    Arc::clone(&pending_batch),
+                    Arc::clone(&source),
+                    Arc::clone(&consecutive_nacks),
+                    batch_id,
+                    SourceBatchResult::Nack,
+                    13,
+                )
+                .await,
+                -1
+            );
+            tokio::time::timeout(Duration::from_secs(1), task)
+                .await
+                .expect("source task did not stop after batch result failure")
+                .expect("source task failed");
+
+            assert_eq!(source.polls.load(Ordering::SeqCst), 1);
+            assert_eq!(
+                *source
+                    .results
+                    .lock()
+                    .unwrap_or_else(PoisonError::into_inner),
+                vec![SourceBatchResult::Nack]
+            );
+        });
+    }
+
+    #[test]
+    fn given_repeated_nacks_when_limit_is_reached_should_stop() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource::default());
+            let consecutive_nacks = AtomicU32::new(0);
+
+            for expected_count in 1..MAX_CONSECUTIVE_NACKS {
+                assert_eq!(
+                    apply_batch_result(
+                        &source,
+                        &consecutive_nacks,
+                        SourceBatchResult::Nack,
+                        23,
+                        MAX_CONSECUTIVE_NACKS,
+                    )
+                    .await,
+                    BatchCompletion::Applied(SourceBatchResult::Nack)
+                );
+                assert_eq!(consecutive_nacks.load(Ordering::Relaxed), expected_count);
+            }
+
+            assert_eq!(
+                apply_batch_result(
+                    &source,
+                    &consecutive_nacks,
+                    SourceBatchResult::Nack,
+                    23,
+                    MAX_CONSECUTIVE_NACKS,
+                )
+                .await,
+                BatchCompletion::Stop
+            );
+            assert_eq!(
+                consecutive_nacks.load(Ordering::Relaxed),
+                MAX_CONSECUTIVE_NACKS
+            );
+        });
+    }
+
+    #[test]
+    fn given_ack_after_nack_should_reset_consecutive_nacks() {
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let source = Arc::new(TestSource::default());
+            let consecutive_nacks = AtomicU32::new(3);
+
+            assert_eq!(
+                apply_batch_result(
+                    &source,
+                    &consecutive_nacks,
+                    SourceBatchResult::Ack,
+                    29,
+                    MAX_CONSECUTIVE_NACKS,
+                )
+                .await,
+                BatchCompletion::Applied(SourceBatchResult::Ack)
+            );
+            assert_eq!(consecutive_nacks.load(Ordering::Relaxed), 0);
+        });
+    }
+
+    #[test]
+    fn given_consecutive_nacks_should_apply_capped_backoff() {
+        let consecutive_nacks = AtomicU32::new(1);
+        let policy = BatchPolicy {
+            nack_retry_delay: Duration::from_millis(100),
+            max_nack_retry_delay: Duration::from_millis(350),
+            ..test_policy()
+        };
+
+        assert_eq!(
+            nack_retry_delay(&consecutive_nacks, policy),
+            Duration::from_millis(100)
+        );
+        consecutive_nacks.store(2, Ordering::Relaxed);
+        assert_eq!(
+            nack_retry_delay(&consecutive_nacks, policy),
+            Duration::from_millis(200)
+        );
+        consecutive_nacks.store(3, Ordering::Relaxed);
+        assert_eq!(
+            nack_retry_delay(&consecutive_nacks, policy),
+            Duration::from_millis(350)
+        );
+        consecutive_nacks.store(4, Ordering::Relaxed);
+        assert_eq!(
+            nack_retry_delay(&consecutive_nacks, policy),
+            Duration::from_millis(350)
+        );
+    }
+}
diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md
index 34989aef0..a795735f4 100644
--- a/core/connectors/sources/README.md
+++ b/core/connectors/sources/README.md
@@ -24,6 +24,9 @@ pub trait Source: Send + Sync {
     /// Invoked every time a batch of messages is produced to the configured stream and topic.
     async fn poll(&self) -> Result<ProducedMessages, Error>;
 
+    /// Invoked after the runtime sends the batch and persists its candidate state.
+    async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error>;
+
     /// Invoked when the source is closed, allowing it to perform any necessary cleanup.
     async fn close(&mut self) -> Result<(), Error>;
 }
@@ -131,7 +134,8 @@ struct State {
 pub struct RandomSource {
     id: u32,
     messages_count: u32,
-    state: Mutex<State>
+    state: Mutex<State>,
+    pending_state: Mutex<Option<State>>,
 }
 ```
 
@@ -162,8 +166,9 @@ impl RandomSource {
 
         RandomSource {
             id,
-            payload_size: config.payload_size.unwrap_or(100),
+            messages_count: config.messages_count.unwrap_or(10),
             state: Mutex::new(State { current_id }),
+            pending_state: Mutex::new(None),
         }
     }
 }
@@ -172,7 +177,7 @@ impl RandomSource {
 We can invoke the expected macro to expose the FFI interface and allow the connector runtime to register the source within the runtime.
 
 ```rust
-source_connector!(TestSource);
+source_connector!(RandomSource);
 ```
 
 At a bare minimum, we need to add the following dependencies to the `Cargo.toml` file to compile the plugin at all:
@@ -205,8 +210,7 @@ impl Source for RandomSource {
 
     async fn poll(&self) -> Result<ProducedMessages, iggy_connector_sdk::Error> {
         sleep(Duration::from_millis(100)).await;
-        let mut state = self.state.lock().await;
-        let current_id = state.current_id;
+        let mut current_id = self.state.lock().await.current_id;
 
         let mut messages = Vec::new();
         for _ in 0..self.messages_count {
@@ -234,19 +238,30 @@ impl Source for RandomSource {
             messages.push(message);
         }
 
-        state.current_id += current_id;
+        let candidate_state = State { current_id };
+        *self.pending_state.lock().await = Some(candidate_state);
         info!(
-            "Generated {} messages by random source connector with ID: {}"
+            "Generated {} messages by random source connector with ID: {}",
             messages.len(),
             self.id,
         );
         Ok(ProducedMessages {
             schema: Schema::Json,
             messages,
-            state: Some(ConnectorState(state.current_id.to_le_bytes().to_vec())),
+            state: Some(ConnectorState(current_id.to_le_bytes().to_vec())),
         })
     }
 
+    async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> {
+        let candidate_state = self.pending_state.lock().await.take();
+        if result == SourceBatchResult::Ack
+            && let Some(candidate_state) = candidate_state
+        {
+            *self.state.lock().await = candidate_state;
+        }
+        Ok(())
+    }
+
     async fn close(&mut self) -> Result<(), Error> {
         info!("Random source connector with ID: {} is closed.", self.id);
         Ok(())
@@ -254,6 +269,8 @@ impl Source for RandomSource {
 }
 ```
 
+The source keeps its committed cursor unchanged while the batch is in flight. An ACK commits the staged cursor; a NACK discards it so the same range can be polled again. Sources that delete or mark upstream rows should stage those operations in the same way.
+
 As you can see, the `ProducedMessage` can be customized to fit your needs, as all the fields will be directly mapped to the existing Iggy message struct.
 
 It's also important to note, that the supported format(s) might vary depending on the connector implementation. For example, you might use `JSON` as the payload format, which can be then easily parsed and processed by downstream components such as data transforms, but at the same time, you could support the other formats and let the user decide which one to use.
diff --git a/core/connectors/sources/random_source/src/lib.rs b/core/connectors/sources/random_source/src/lib.rs
index 1a68e3f4a..908640d3f 100644
--- a/core/connectors/sources/random_source/src/lib.rs
+++ b/core/connectors/sources/random_source/src/lib.rs
@@ -17,7 +17,8 @@
 
 use async_trait::async_trait;
 use iggy_connector_sdk::{
-    ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector,
+    ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source,
+    source::SourceBatchResult, source_connector,
 };
 use rand::{
     RngExt,
@@ -41,6 +42,7 @@ pub struct RandomSource {
     messages_range: (u32, u32),
     payload_size: u32,
     state: Mutex<State>,
+    pending_state: Mutex<Option<State>>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -51,7 +53,7 @@ pub struct RandomSourceConfig {
     payload_size: Option<u32>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct State {
     messages_produced: usize,
 }
@@ -81,6 +83,7 @@ impl RandomSource {
             state: Mutex::new(restored_state.unwrap_or(State {
                 messages_produced: 0,
             })),
+            pending_state: Mutex::new(None),
         }
     }
 
@@ -88,11 +91,14 @@ impl RandomSource {
         ConnectorState::serialize(state, CONNECTOR_NAME, self.id)
     }
 
-    fn generate_messages(&self) -> Vec<ProducedMessage> {
+    fn generate_messages(&self, remaining: Option<usize>) -> Vec<ProducedMessage> {
         let mut messages = Vec::new();
         let mut rng = rand::rng();
+        let messages_count = rng
+            .sample(Uniform::new(self.messages_range.0, self.messages_range.1).unwrap())
+            as usize;
         let messages_count =
-            rng.sample(Uniform::new(self.messages_range.0, self.messages_range.1).unwrap());
+            remaining.map_or(messages_count, |remaining| messages_count.min(remaining));
         for _ in 0..messages_count {
             let record = Record {
                 id: Uuid::new_v4(),
@@ -147,9 +153,9 @@ impl Source for RandomSource {
 
     async fn poll(&self) -> Result<ProducedMessages, iggy_connector_sdk::Error> {
         sleep(self.interval).await;
-        let mut state = self.state.lock().await;
+        let messages_produced = self.state.lock().await.messages_produced;
         if let Some(max_count) = self.max_count
-            && state.messages_produced >= max_count
+            && messages_produced >= max_count
         {
             info!(
                 "Reached max number of {max_count} messages for {CONNECTOR_NAME} connector with ID: {}",
@@ -158,28 +164,45 @@ impl Source for RandomSource {
             return Ok(ProducedMessages {
                 schema: Schema::Json,
                 messages: vec![],
-                state: self.serialize_state(&state),
+                state: None,
             });
         }
 
-        let messages = self.generate_messages();
-        state.messages_produced += messages.len();
+        let remaining = self
+            .max_count
+            .map(|max_count| max_count.saturating_sub(messages_produced));
+        let messages = self.generate_messages(remaining);
+        let candidate_state = State {
+            messages_produced: messages_produced + messages.len(),
+        };
+        let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| {
+            Error::Serialization("failed to serialize random source state".to_string())
+        })?;
+        *self.pending_state.lock().await = Some(candidate_state.clone());
         info!(
             "{CONNECTOR_NAME} connector with ID: {} generated {} messages. Total produced: {}",
             self.id,
             messages.len(),
-            state.messages_produced
+            candidate_state.messages_produced
         );
 
-        let persisted_state = self.serialize_state(&state);
-
         Ok(ProducedMessages {
             schema: Schema::Json,
             messages,
-            state: persisted_state,
+            state: Some(persisted_state),
         })
     }
 
+    async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> {
+        let candidate_state = self.pending_state.lock().await.take();
+        if result == SourceBatchResult::Ack
+            && let Some(candidate_state) = candidate_state
+        {
+            *self.state.lock().await = candidate_state;
+        }
+        Ok(())
+    }
+
     async fn close(&mut self) -> Result<(), Error> {
         let state = self.state.lock().await;
         info!(
@@ -272,11 +295,45 @@ mod tests {
             messages_produced: 42,
         };
 
-        let connector_state = src.serialize_state(&state);
-        assert!(connector_state.is_some());
-
-        let bytes = connector_state.unwrap().0;
+        let bytes = src
+            .serialize_state(&state)
+            .expect("state should be serializable")
+            .0;
         let restored: State = rmp_serde::from_slice(&bytes).expect("Failed to deserialize state");
         assert_eq!(restored.messages_produced, 42);
     }
+
+    #[test]
+    fn given_nack_when_batch_is_staged_should_keep_committed_state() {
+        let src = RandomSource::new(1, test_config(), None);
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let produced = src.poll().await.expect("poll should succeed");
+            assert!(!produced.messages.is_empty());
+
+            src.on_batch_result(SourceBatchResult::Nack)
+                .await
+                .expect("NACK should be applied");
+
+            assert_eq!(src.state.lock().await.messages_produced, 0);
+            assert!(src.pending_state.lock().await.is_none());
+        });
+    }
+
+    #[test]
+    fn given_ack_when_batch_is_staged_should_commit_candidate_state() {
+        let src = RandomSource::new(1, test_config(), None);
+        let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime");
+        runtime.block_on(async {
+            let produced = src.poll().await.expect("poll should succeed");
+            let count = produced.messages.len();
+
+            src.on_batch_result(SourceBatchResult::Ack)
+                .await
+                .expect("ACK should be applied");
+
+            assert_eq!(src.state.lock().await.messages_produced, count);
+            assert!(src.pending_state.lock().await.is_none());
+        });
+    }
 }
diff --git a/core/integration/tests/connectors/random/random_source.rs b/core/integration/tests/connectors/random/random_source.rs
index 9203f8711..772a2a225 100644
--- a/core/integration/tests/connectors/random/random_source.rs
+++ b/core/integration/tests/connectors/random/random_source.rs
@@ -16,8 +16,19 @@
 // under the License.
 
 use crate::connectors::random_source_liveness;
+use iggy_connector_sdk::api::{ConnectorRuntimeStats, ConnectorStatus};
 use integration::harness::seeds;
 use integration::iggy_harness;
+use reqwest::Client;
+use std::path::Path;
+use std::time::Duration;
+use tokio::time::{sleep, timeout};
+
+const API_KEY: &str = "test-api-key";
+const SOURCE_KEY: &str = "random";
+const RETRY_INTERVAL: Duration = Duration::from_millis(100);
+const STATE_STABILITY_WINDOW: Duration = Duration::from_secs(1);
+const WAIT_TIMEOUT: Duration = Duration::from_secs(10);
 
 #[iggy_harness(
     server(connectors_runtime(config_path = "tests/connectors/random/source.toml")),
@@ -26,3 +37,115 @@ use integration::iggy_harness;
 async fn random_source_produces_messages(harness: &TestHarness) {
     random_source_liveness::assert_produces_messages(harness).await;
 }
+
+#[iggy_harness(
+    cluster_nodes = 1,
+    server(connectors_runtime(config_path = "tests/connectors/random/source.toml")),
+    seed = seeds::connector_stream
+)]
+async fn state_save_failure_preserves_state_and_source_recovers(harness: &TestHarness) {
+    let state_path = harness
+        .connectors_runtime()
+        .expect("connectors runtime")
+        .state_path()
+        .join("source_random.state");
+    wait_for_state_file(&state_path).await;
+    let api_url = harness
+        .connectors_runtime()
+        .expect("connectors runtime")
+        .http_url();
+    let http = Client::new();
+    let errors_before_failure = source_errors(&http, &api_url).await;
+    let state_dir = state_path.parent().expect("source state directory");
+    let unavailable_state_dir = state_dir.with_extension("unavailable");
+    let unavailable_state_path = unavailable_state_dir.join("source_random.state");
+
+    tokio::fs::rename(state_dir, &unavailable_state_dir)
+        .await
+        .expect("source state directory should become unavailable");
+    let state_before_failure = tokio::fs::read(&unavailable_state_path)
+        .await
+        .expect("source state should remain readable");
+    wait_for_source_error_after(&http, &api_url, errors_before_failure).await;
+
+    sleep(STATE_STABILITY_WINDOW).await;
+    assert_eq!(
+        tokio::fs::read(&unavailable_state_path)
+            .await
+            .expect("source state should remain readable"),
+        state_before_failure,
+        "NACKed batches must not advance persisted source state"
+    );
+
+    tokio::fs::rename(&unavailable_state_dir, state_dir)
+        .await
+        .expect("source state directory should become available");
+    timeout(WAIT_TIMEOUT, async {
+        loop {
+            if tokio::fs::read(&state_path)
+                .await
+                .is_ok_and(|state| state != state_before_failure)
+            {
+                break;
+            }
+            sleep(RETRY_INTERVAL).await;
+        }
+    })
+    .await
+    .expect("source did not complete another batch after state storage recovery");
+
+    random_source_liveness::assert_produces_messages(harness).await;
+}
+
+async fn wait_for_state_file(state_path: &Path) {
+    timeout(Duration::from_secs(5), async {
+        while !state_path.exists() {
+            sleep(Duration::from_millis(50)).await;
+        }
+    })
+    .await
+    .expect("source state file was not created");
+}
+
+async fn source_errors(http: &Client, api_url: &str) -> u64 {
+    let stats = http
+        .get(format!("{api_url}/stats"))
+        .header("api-key", API_KEY)
+        .send()
+        .await
+        .expect("runtime stats should be available")
+        .json::<ConnectorRuntimeStats>()
+        .await
+        .expect("runtime stats should be valid");
+    stats
+        .connectors
+        .iter()
+        .find(|connector| connector.key == SOURCE_KEY)
+        .expect("random source stats should be present")
+        .errors
+}
+
+async fn wait_for_source_error_after(http: &Client, api_url: &str, previous_errors: u64) {
+    timeout(WAIT_TIMEOUT, async {
+        loop {
+            if let Ok(response) = http
+                .get(format!("{api_url}/stats"))
+                .header("api-key", API_KEY)
+                .send()
+                .await
+                && let Ok(stats) = response.json::<ConnectorRuntimeStats>().await
+                && let Some(source) = stats
+                    .connectors
+                    .iter()
+                    .find(|connector| connector.key == SOURCE_KEY)
+                && source.status == ConnectorStatus::Error
+                && source.errors > previous_errors
+            {
+                break;
+            }
+            sleep(RETRY_INTERVAL).await;
+        }
+    })
+    .await
+    .expect("random source did not report a state-save failure")
+}