You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/07/07 14:06:30 UTC

(iggy) 01/03: watermark

This is an automated email from the ASF dual-hosted git repository.

numinnex pushed a commit to branch fix_reconnect_panic
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit c103effc1409642bcde2b32f17f793494e9f9134
Author: Grzegorz Koszyk <g....@gmail.com>
AuthorDate: Tue Jul 7 08:43:42 2026 +0200

    watermark
---
 core/consensus/src/impls.rs                   | 53 +++++++++++++++---
 core/integration/tests/server/purge_delete.rs | 21 ++------
 core/metadata/src/impls/recovery.rs           | 75 ++++++++++++++++++++++++--
 core/partitions/src/iggy_partition.rs         | 31 ++++++++++-
 core/sdk/src/quic/quic_client.rs              |  7 ---
 core/sdk/src/session.rs                       | 12 +++++
 core/sdk/src/tcp/tcp_client.rs                |  7 ---
 core/server-ng/src/bootstrap.rs               | 78 +++++++++++++++++----------
 core/server-ng/src/partition_helpers.rs       | 35 ++++++------
 core/server-ng/src/server_error.rs            | 12 -----
 core/shard/src/lib.rs                         | 78 ++++++++++++++++++++++++---
 core/shard/src/router.rs                      |  5 ++
 12 files changed, 310 insertions(+), 104 deletions(-)

diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index 94be74bf1..d17a48c88 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -546,6 +546,19 @@ pub enum Status {
 
 /// Actions to be taken by the caller after processing a VSR event.
 #[derive(Debug, Clone)]
+pub enum CommitOutcome {
+    /// Heartbeat ignored (stale view, foreign sender, wrong status, or replay).
+    Ignored,
+    /// Heartbeat accepted; `commit_max` unchanged.
+    Accepted,
+    /// Heartbeat accepted and `commit_max` advanced; run `commit_journal`.
+    Advanced,
+    /// A replica is still heartbeating an older view in which it was
+    /// primary; this replica is the current view's primary and should
+    /// broadcast `StartView` so the stale replica adopts the view.
+    RespondStartView,
+}
+
 pub enum VsrAction {
     /// Send `StartViewChange` to all replicas.
     SendStartViewChange { view: u32, namespace: u64 },
@@ -1721,6 +1734,14 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
         }
 
         // Accept the StartView and transition to normal
+        tracing::info!(
+            replica = self.replica,
+            old_view = self.view.get(),
+            new_view = msg_view,
+            op = msg_op,
+            commit = msg_commit,
+            "adopting view from StartView"
+        );
         self.view.set(msg_view);
         self.log_view.set(msg_view);
         self.status.set(Status::Normal);
@@ -1791,29 +1812,45 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
     ///
     /// # Panics
     /// If `header.namespace` does not match this replica's namespace.
-    pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> bool {
+    pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> CommitOutcome {
         assert_eq!(
             header.namespace, self.namespace,
             "Commit routed to wrong group"
         );
 
         if self.is_primary() {
-            return false;
+            // A heartbeat from the primary of an OLDER view means that
+            // replica missed our view change entirely -- typically it
+            // restarted while the view advanced and recovered the stale
+            // view from its journal (there is no durable view watermark),
+            // so the SVC/DVC/SV exchange never reached it. Left alone it
+            // wedges: it drops our newer-view traffic as foreign and we
+            // drop its stale prepares, while its live heartbeats keep its
+            // backups from electing anyone. Re-announcing the current view
+            // lets its `handle_start_view` adopt the view and cancel its
+            // stale pipeline.
+            if self.status.get() == Status::Normal
+                && header.view < self.view.get()
+                && header.replica == self.primary_index(header.view)
+            {
+                return CommitOutcome::RespondStartView;
+            }
+            return CommitOutcome::Ignored;
         }
 
         if self.status.get() != Status::Normal {
-            return false;
+            return CommitOutcome::Ignored;
         }
 
         if header.view != self.view.get() {
-            return false;
+            return CommitOutcome::Ignored;
         }
 
         // TODO: Once connection-level peer verification is added promote
         // this to an assert, the network layer would guarantee the sender
         // matches header.replica.
         if header.replica != self.primary_index(header.view) {
-            return false;
+            return CommitOutcome::Ignored;
         }
 
         // Only accept heartbeats with a strictly newer timestamp to prevent
@@ -1827,7 +1864,11 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
 
         let old_commit_max = self.commit_max.get();
         self.advance_commit_max(header.commit);
-        self.commit_max.get() > old_commit_max
+        if self.commit_max.get() > old_commit_max {
+            CommitOutcome::Advanced
+        } else {
+            CommitOutcome::Accepted
+        }
     }
 
     /// Complete view change as the new primary after collecting DVC quorum.
diff --git a/core/integration/tests/server/purge_delete.rs b/core/integration/tests/server/purge_delete.rs
index 92dfe3539..dc226470f 100644
--- a/core/integration/tests/server/purge_delete.rs
+++ b/core/integration/tests/server/purge_delete.rs
@@ -19,18 +19,14 @@ use crate::server::scenarios::purge_delete_scenario;
 use integration::iggy_harness;
 use test_case::test_matrix;
 
-// Legacy-only: asserts the exact on-disk layout ([0, 7, 14, 21], 7 messages per
-// 5KiB segment) and per-file byte sizes throughout. server-ng's per-message
-// framing yields a different layout, so the scenario's offset anchors do not
-// hold under vsr. Port to a framing-agnostic form to re-enable.
-#[cfg(not(feature = "vsr"))]
 #[iggy_harness(server(
     segment.size = "5KiB",
     segment.cache_indexes = ["all", "none", "open_segment"],
     partition.messages_required_to_save = "1",
     partition.enforce_fsync = "true",
 ))]
-#[test_matrix([restart_off(), restart_on()])]
+#[cfg_attr(not(feature = "vsr"), test_matrix([restart_off(), restart_on()]))]
+#[cfg_attr(feature = "vsr", test_matrix([restart_off()]))]
 async fn should_delete_segments_and_validate_filesystem(
     harness: &mut TestHarness,
     restart_server: bool,
@@ -53,13 +49,6 @@ async fn should_delete_segments_without_consumers(harness: &mut TestHarness, res
     purge_delete_scenario::run_no_consumers(harness, restart_server).await;
 }
 
-// vsr-gated: asserts a segment is freed *exactly* when the polled offset
-// reaches its end — a synchronous-commit assumption. Under vsr the
-// consumer-group auto-commit lags the poll, so the per-message timing does not
-// hold (the barrier itself is correct: the server retains un-consumed
-// segments, verified by `min_committed_offset` + its runtime diag). Legacy-only
-// until redesigned to sync on the committed offset for async commit.
-#[cfg(not(feature = "vsr"))]
 #[iggy_harness(server(
     segment.size = "5KiB",
     segment.cache_indexes = ["all", "none", "open_segment"],
@@ -73,16 +62,14 @@ async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarnes
     purge_delete_scenario::run_consumer_group_barrier(&client, &data_path).await;
 }
 
-// Legacy-only: pins the [0, 7, 14, 21] layout across a multi-consumer barrier.
-// server-ng framing differs; port to framing-agnostic to re-enable.
-#[cfg(not(feature = "vsr"))]
 #[iggy_harness(server(
     segment.size = "5KiB",
     segment.cache_indexes = ["all", "none", "open_segment"],
     partition.messages_required_to_save = "1",
     partition.enforce_fsync = "true",
 ))]
-#[test_matrix([restart_off(), restart_on()])]
+#[cfg_attr(not(feature = "vsr"), test_matrix([restart_off(), restart_on()]))]
+#[cfg_attr(feature = "vsr", test_matrix([restart_off()]))]
 async fn should_block_deletion_until_all_consumers_pass_segment(
     harness: &mut TestHarness,
     restart_server: bool,
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs
index 4268c1633..91325aaa4 100644
--- a/core/metadata/src/impls/recovery.rs
+++ b/core/metadata/src/impls/recovery.rs
@@ -88,7 +88,16 @@ pub struct RecoveredMetadata<M> {
     pub mux_stm: M,
     /// `None` means no snapshot existed and no journal entries were replayed.
     /// `Some(op)` is the highest op applied, either from the snapshot or journal replay.
+    ///
+    /// Only the committed prefix is applied: an entry's `commit` header field
+    /// carries the primary's commit watermark when the prepare was sent, and
+    /// journaled does not imply committed.
     pub last_applied_op: Option<u64>,
+    /// Highest op present in the journal, `None` when it is empty. Ops in
+    /// `(last_applied_op, last_journaled_op]` are prepared-but-uncommitted:
+    /// they stay journal-only until the recovered primary re-replicates them
+    /// (or a backup sees the commit point advance past them).
+    pub last_journaled_op: Option<u64>,
 }
 
 /// Recover metadata state from disk.
@@ -138,12 +147,29 @@ where
     // must repair or truncate the WAL before the node can boot again.
     let headers_to_replay = journal.iter_headers_from(replay_from);
 
+    // Committed watermark: the `commit` field of each journaled prepare is
+    // the primary's commit point when it was sent (monotone in op order), so
+    // the last entry carries the highest commit this WAL can prove. Ops above
+    // it were prepared but not provably committed; applying them here would
+    // fabricate commit knowledge (a crashed suffix may have been discarded by
+    // a view change) and would hide the suffix from re-replication.
+    let snapshot_floor = snapshot.as_ref().map_or(0, IggySnapshot::sequence_number);
+    let commit_watermark = headers_to_replay
+        .last()
+        .map_or(snapshot_floor, |header| header.commit.max(snapshot_floor));
+
     let mut last_applied_op: Option<u64> = None;
+    let mut last_journaled_op: Option<u64> = None;
     for header in &headers_to_replay {
         // TODO: Check hash chain integrity against `previous_header`. On a
         // same-view break, stop replay here and mark the remaining entries for
         // repair via VSR instead of panicking.
 
+        last_journaled_op = Some(header.op);
+        if header.op > commit_watermark {
+            continue;
+        }
+
         let entry = journal.entry_at(header).await?.ok_or_else(|| {
             RecoveryError::Io(std::io::Error::new(
                 std::io::ErrorKind::InvalidData,
@@ -161,6 +187,7 @@ where
         snapshot,
         mux_stm,
         last_applied_op,
+        last_journaled_op,
     })
 }
 
@@ -180,6 +207,12 @@ mod tests {
     const HEADER_SIZE: usize = size_of::<PrepareHeader>();
 
     fn make_prepare(op: u64, body_size: usize) -> Message<PrepareHeader> {
+        make_prepare_with_commit(op, op.saturating_sub(1), body_size)
+    }
+
+    /// A prepare as a live primary stamps it: `commit` carries the primary's
+    /// commit point when the prepare is sent.
+    fn make_prepare_with_commit(op: u64, commit: u64, body_size: usize) -> Message<PrepareHeader> {
         let total_size = HEADER_SIZE + body_size;
         let mut buffer = Owned::<4096>::zeroed(total_size);
         let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(
@@ -188,6 +221,7 @@ mod tests {
         header.size = total_size as u32;
         header.command = Command2::Prepare;
         header.op = op;
+        header.commit = commit;
         header.operation = Operation::CreateStream;
         Message::try_from(buffer).unwrap()
     }
@@ -241,10 +275,44 @@ mod tests {
 
         let recovered = recover::<TestStm>(dir.path()).await.unwrap();
         assert!(recovered.snapshot.is_none());
-        assert_eq!(recovered.last_applied_op, Some(3));
+        // Op 3's entry proves commit=2; op 3 itself is journaled but not
+        // provably committed, so it stays journal-only.
+        assert_eq!(recovered.last_applied_op, Some(2));
+        assert_eq!(recovered.last_journaled_op, Some(3));
         assert_eq!(recovered.journal.last_op(), Some(3));
     }
 
+    #[compio::test]
+    async fn recover_applies_only_the_committed_prefix() {
+        let dir = tempdir().unwrap();
+        let metadata_dir = dir.path().join("metadata");
+        std::fs::create_dir_all(&metadata_dir).unwrap();
+
+        // Ops 1-5 journaled, but the last entry proves commit only up to 3:
+        // the primary crashed before 4 and 5 reached quorum.
+        {
+            let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0)
+                .await
+                .unwrap();
+            for op in 1..=4u64 {
+                journal
+                    .append(make_prepare_with_commit(op, op.saturating_sub(1), 32))
+                    .await
+                    .unwrap();
+            }
+            journal
+                .append(make_prepare_with_commit(5, 3, 32))
+                .await
+                .unwrap();
+            journal.storage_ref().fsync().await.unwrap();
+        }
+
+        let recovered = recover::<TestStm>(dir.path()).await.unwrap();
+        assert_eq!(recovered.last_applied_op, Some(3));
+        assert_eq!(recovered.last_journaled_op, Some(5));
+        assert_eq!(recovered.journal.last_op(), Some(5));
+    }
+
     #[compio::test]
     async fn recover_snapshot_plus_journal() {
         let dir = tempdir().unwrap();
@@ -269,8 +337,9 @@ mod tests {
         }
 
         let recovered = recover::<TestStm>(dir.path()).await.unwrap();
-        // Should replay ops 6-10 (snapshot was at 5)
-        assert_eq!(recovered.last_applied_op, Some(10));
+        // Replays ops 6-9 (snapshot at 5; op 10's entry proves commit=9).
+        assert_eq!(recovered.last_applied_op, Some(9));
+        assert_eq!(recovered.last_journaled_op, Some(10));
         assert_eq!(
             recovered
                 .snapshot
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 7373577d2..c171d1174 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -1726,8 +1726,35 @@ where
         self.append_messages(message).await.map(|_| ())
     }
 
-    #[allow(clippy::too_many_lines)]
     async fn commit_messages(&mut self, config: &PartitionsConfig) -> Result<(), IggyError> {
+        self.commit_messages_inner(config, false).await
+    }
+
+    /// Flush the committed journal prefix to segment storage regardless of
+    /// the `messages_required_to_save` thresholds.
+    ///
+    /// Shutdown-path counterpart of the commit-time persist gate: a graceful
+    /// stop must not lose committed messages that were still resident in the
+    /// in-memory journal (consumer offsets are persisted eagerly, so losing
+    /// the messages would fail recovery with an offset ahead of the data).
+    ///
+    /// # Errors
+    ///
+    /// Returns [`IggyError`] when writing the committed batches or their
+    /// index entries to segment storage fails.
+    pub async fn flush_committed_messages(
+        &mut self,
+        config: &PartitionsConfig,
+    ) -> Result<(), IggyError> {
+        self.commit_messages_inner(config, true).await
+    }
+
+    #[allow(clippy::too_many_lines)]
+    async fn commit_messages_inner(
+        &mut self,
+        config: &PartitionsConfig,
+        force: bool,
+    ) -> Result<(), IggyError> {
         let write_lock = self.write_lock.clone();
         let _guard = write_lock.lock().await;
 
@@ -1748,7 +1775,7 @@ where
             >= config.size_of_messages_required_to_save.as_bytes_u64();
         let should_persist =
             is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded;
-        if !should_persist {
+        if !force && !should_persist {
             return Ok(());
         }
 
diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs
index 8b41ac790..6cc24cd0b 100644
--- a/core/sdk/src/quic/quic_client.rs
+++ b/core/sdk/src/quic/quic_client.rs
@@ -177,13 +177,6 @@ impl BinaryTransport for QuicClient {
         #[cfg(feature = "vsr")]
         if skip_auto_login {
             *self.skip_auto_login_once.lock().await = true;
-            // The replayed login/register must mint a fresh Register: the failed
-            // attempt already consumed the one-shot register request id and may
-            // have half-bound the session.
-            *self
-                .consensus_session
-                .lock()
-                .expect("consensus session mutex poisoned") = ConsensusSession::new();
         }
         let server_address = self.current_server_address.lock().await.to_string();
         info!(
diff --git a/core/sdk/src/session.rs b/core/sdk/src/session.rs
index ac56c0b33..33bbaaa51 100644
--- a/core/sdk/src/session.rs
+++ b/core/sdk/src/session.rs
@@ -98,6 +98,18 @@ impl ConsensusSession {
         self.session.is_some()
     }
 
+    /// Whether a login must mint a fresh session before registering.
+    ///
+    /// The register request id is one-shot per session, so any re-login
+    /// (reconnect auto-login or a deliberate second login) on a consumed or
+    /// bound session requires recreating the session first: new random
+    /// `client_id`, unambiguous re-register; the server evicts the abandoned
+    /// entry (see the module docs).
+    #[must_use]
+    pub fn needs_fresh_register(&self) -> bool {
+        self.register_consumed || self.is_bound()
+    }
+
     /// Bind the session after register commits through consensus.
     /// The `session` value is the commit op number from the server's reply.
     ///
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs
index d69ffd2ae..6200179dd 100644
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@ -179,13 +179,6 @@ impl BinaryTransport for TcpClient {
         #[cfg(feature = "vsr")]
         if skip_auto_login {
             *self.skip_auto_login_once.lock().await = true;
-            // The replayed login/register must mint a fresh Register: the
-            // failed attempt already consumed the one-shot register request
-            // id and may have half-bound the session.
-            *self
-                .consensus_session
-                .lock()
-                .expect("consensus session mutex poisoned") = ConsensusSession::new();
         }
 
         {
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index ca7710148..d93b93feb 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -34,7 +34,9 @@ use configs::server_ng::ServerNgConfig;
 use configs::sharding::{
     INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX,
 };
-use consensus::{LocalPipeline, MetadataHandle, PartitionsHandle, Sequencer, VsrConsensus};
+use consensus::{
+    LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, VsrConsensus,
+};
 // `try_send` / `try_recv` resolve through these traits on `MAsyncTx` /
 // `MAsyncRx`; the metadata-handoff loops below depend on the
 // non-blocking variants for cancel-safe shutdown polling.
@@ -819,6 +821,7 @@ async fn shard_main(
                     recovered.journal,
                     recovered.snapshot,
                     recovered.last_applied_op,
+                    recovered.last_journaled_op,
                 )),
             )
         }
@@ -838,12 +841,14 @@ async fn shard_main(
     // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`,
     // so peer shards have no caller that reads `journal` or `snapshot`.
     let (metadata_consensus, journal_for_metadata, snapshot_for_metadata) =
-        if let Some((journal, snapshot, last_applied_op)) = owner_state {
-            let restored_op = last_applied_op
-                .unwrap_or_else(|| snapshot.as_ref().map_or(0, IggySnapshot::sequence_number));
+        if let Some((journal, snapshot, last_applied_op, last_journaled_op)) = owner_state {
+            let snapshot_floor = snapshot.as_ref().map_or(0, IggySnapshot::sequence_number);
+            let commit_watermark = last_applied_op.unwrap_or(snapshot_floor);
+            let restored_op = last_journaled_op.unwrap_or(snapshot_floor);
             let consensus = restore_metadata_consensus(
                 &journal,
                 restored_op,
+                commit_watermark,
                 topology.cluster_id,
                 topology.self_replica_id,
                 topology.replica_count,
@@ -1482,6 +1487,7 @@ async fn build_shard_for_thread(
 fn restore_metadata_consensus(
     journal: &PrepareJournal,
     restored_op: u64,
+    commit_watermark: u64,
     cluster_id: u128,
     self_replica_id: u8,
     replica_count: u8,
@@ -1506,36 +1512,50 @@ fn restore_metadata_consensus(
 
     consensus.init();
     consensus.sequencer().set_sequence(restored_op);
-    // TODO(hubcio): clustered bootstrap does not persist a durable
-    // (view, commit_op) watermark, so we collapse commit_min/commit_max to
-    // `restored_op` (= last journaled op). VSR consequence: on a view
-    // change after partial recovery, a replica that came back with a
-    // commit_max below the cluster's true commit_min will accept stale
-    // prepares as new and overwrite already-committed log entries
-    // (split-brain on the committed prefix).
-    //
-    // Fix direction: persist (view, commit_op) on the journal-header path
-    // (`core/journal/src/prepare_journal.rs` PrepareHeader already carries
-    // `view`; extend with `commit_op` or add a sidecar watermark file
-    // updated on every commit), seed `restore_commit_state(min, max)`
-    // from durable state on recovery, and refuse boot if the gap exceeds
-    // a configurable threshold. Tracked under the "durable
-    // PartitionJournal + durable (view, commit_op) watermark" milestone
-    // named in the multi-shard wiring commit body.
+    // The commit point is restored from the WAL's embedded watermark (each
+    // journaled prepare carries the primary's commit at send time), NOT from
+    // the journal head: journaled does not imply committed, and claiming
+    // commit for the un-quorum'd tail both risks split-brain on a later view
+    // change and starves the tail of re-replication (it would live in no
+    // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined
+    // below when this replica is the recovered view's primary.
     //
-    // Reproducible in `core/simulator` once it grows a restart-from-disk
-    // path: today `SimNetwork::enable_process` only un-disables links
-    // without replaying `SimJournal` + `SimSnapshot` through
-    // `restore_metadata_consensus`, so the flatten cannot trip. Add a
-    // crash-restart-replay primitive in the sim, then write a scenario
-    // that commits an op on the primary, crashes the primary mid-
-    // replicate-ack, triggers a view change, and asserts the recovered
-    // replica refuses to re-accept the committed prepare.
-    consensus.restore_commit_state(restored_op, restored_op);
+    // TODO(hubcio): the watermark is a lower bound (the last entry stamps
+    // the commit point as of its send). Persisting an explicit (view,
+    // commit_op) watermark on the commit path would tighten recovery and
+    // allow refusing boot on an excessive gap; a backup that recovered a
+    // LONGER tail than the cluster's primary still needs uncommitted-suffix
+    // truncation when conflicting ops arrive (message repair milestone).
+    consensus.restore_commit_state(commit_watermark, commit_watermark);
     if let Some(header) = last_header {
         consensus.set_last_prepare_checksum(header.checksum);
     }
 
+    // Re-pipeline the prepared-but-uncommitted suffix so the primary's
+    // retransmit machinery re-replicates it and quorum can (re-)commit it.
+    // A backup's suffix stays journal-only: the primary's traffic either
+    // confirms it (re-forward + re-ack path) or supersedes it.
+    if consensus.is_primary() && commit_watermark < restored_op {
+        info!(
+            commit_watermark,
+            restored_op, "re-pipelining recovered uncommitted metadata suffix"
+        );
+        let mut pipeline = consensus.pipeline().borrow_mut();
+        #[allow(clippy::cast_possible_truncation)]
+        for op in (commit_watermark + 1)..=restored_op {
+            let Some(header) = journal.header(op as usize) else {
+                warn!(
+                    op,
+                    "recovered journal suffix has a gap; stopping re-pipeline"
+                );
+                break;
+            };
+            let mut entry = PipelineEntry::new(*header);
+            entry.add_ack(self_replica_id);
+            pipeline.push(entry);
+        }
+    }
+
     consensus
 }
 
diff --git a/core/server-ng/src/partition_helpers.rs b/core/server-ng/src/partition_helpers.rs
index a6aa71fb1..b544e9b4b 100644
--- a/core/server-ng/src/partition_helpers.rs
+++ b/core/server-ng/src/partition_helpers.rs
@@ -41,7 +41,7 @@ use std::path::Path;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
-use tracing::error;
+use tracing::{error, warn};
 
 /// Validate that a namespace fits within the static caps declared in
 /// `config.extra.namespace`.
@@ -172,10 +172,9 @@ pub async fn create_partition_file_hierarchy(
 ///
 /// # Errors
 ///
-/// Returns [`ServerNgError::RecoveredConsumerOffsetOutOfBounds`] when a
-/// stored offset exceeds `current_offset`, or
-/// [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files exist
-/// but fail to decode.
+/// Returns [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files
+/// exist but fail to decode. A stored offset ahead of `current_offset` is
+/// clamped (with a warning), not an error.
 pub fn configure_consumer_offsets(
     partition: &mut IggyPartition<Rc<IggyMessageBus>>,
     config: &ServerNgConfig,
@@ -207,15 +206,20 @@ pub fn configure_consumer_offsets(
         for offset in loaded_consumer_offsets {
             let recovered_offset = offset.offset.load(Ordering::Relaxed);
             if recovered_offset > current_offset {
-                return Err(ServerNgError::RecoveredConsumerOffsetOutOfBounds {
-                    consumer_kind: "consumer",
-                    consumer_id: offset.consumer_id as usize,
-                    offset: recovered_offset,
+                // A crash can persist an offset ahead of the flushed data
+                // (offsets are stored eagerly, messages flush later). Clamp to
+                // the recovered head so the consumer resumes instead of being
+                // stuck polling past the log; mirrors the legacy contract.
+                warn!(
+                    consumer_id = offset.consumer_id,
+                    recovered_offset,
                     current_offset,
                     stream_id,
                     topic_id,
                     partition_id,
-                });
+                    "recovered consumer offset ahead of partition data; clamping"
+                );
+                offset.offset.store(current_offset, Ordering::Relaxed);
             }
             guard.insert(offset.consumer_id as usize, offset);
         }
@@ -233,15 +237,16 @@ pub fn configure_consumer_offsets(
         for (group_id, offset) in loaded_group_offsets {
             let recovered_offset = offset.offset.load(Ordering::Relaxed);
             if recovered_offset > current_offset {
-                return Err(ServerNgError::RecoveredConsumerOffsetOutOfBounds {
-                    consumer_kind: "consumer group",
-                    consumer_id: group_id.0,
-                    offset: recovered_offset,
+                warn!(
+                    consumer_group_id = group_id.0,
+                    recovered_offset,
                     current_offset,
                     stream_id,
                     topic_id,
                     partition_id,
-                });
+                    "recovered consumer group offset ahead of partition data; clamping"
+                );
+                offset.offset.store(current_offset, Ordering::Relaxed);
             }
             guard.insert(group_id, offset);
         }
diff --git a/core/server-ng/src/server_error.rs b/core/server-ng/src/server_error.rs
index 5dc443a9d..041420812 100644
--- a/core/server-ng/src/server_error.rs
+++ b/core/server-ng/src/server_error.rs
@@ -160,18 +160,6 @@ pub enum ServerNgError {
         #[source]
         source: Box<iggy_common::IggyError>,
     },
-    #[error(
-        "recovered {consumer_kind} offset {offset} for id {consumer_id} exceeds current_offset {current_offset} in stream {stream_id}, topic {topic_id}, partition {partition_id}"
-    )]
-    RecoveredConsumerOffsetOutOfBounds {
-        consumer_kind: &'static str,
-        consumer_id: usize,
-        offset: u64,
-        current_offset: u64,
-        stream_id: usize,
-        topic_id: usize,
-        partition_id: usize,
-    },
     #[error(
         "recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})"
     )]
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 3b6b4e60b..7b2b48bc1 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -27,8 +27,8 @@ pub use config::CoordinatorConfig;
 #[cfg(any(test, feature = "simulator"))]
 use consensus::LocalPipeline;
 use consensus::{
-    Consensus, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane, PlaneKind, Sequencer,
-    VsrAction, VsrConsensus,
+    CommitOutcome, Consensus, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane,
+    PlaneKind, Sequencer, VsrAction, VsrConsensus,
 };
 use iggy_binary_protocol::{
     Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader,
@@ -1768,8 +1768,12 @@ where
         if let Some(ref consensus) = planes.0.consensus
             && consensus.namespace() == header.namespace
         {
-            if consensus.handle_commit(&header) {
-                planes.0.commit_journal().await;
+            match consensus.handle_commit(&header) {
+                CommitOutcome::Advanced => planes.0.commit_journal().await,
+                CommitOutcome::RespondStartView => {
+                    respond_start_view::<B, _, MJ>(consensus).await;
+                }
+                CommitOutcome::Accepted | CommitOutcome::Ignored => {}
             }
             return;
         }
@@ -1785,8 +1789,12 @@ where
             return;
         };
         let consensus = partition.consensus();
-        if consensus.handle_commit(&header) {
-            partition.commit_journal(config).await;
+        match consensus.handle_commit(&header) {
+            CommitOutcome::Advanced => partition.commit_journal(config).await,
+            CommitOutcome::RespondStartView => {
+                respond_start_view::<B, _, MJ>(consensus).await;
+            }
+            CommitOutcome::Accepted | CommitOutcome::Ignored => {}
         }
     }
 
@@ -1825,6 +1833,34 @@ where
         }
     }
 
+    /// Flush every owned partition's committed journal prefix to segment
+    /// storage. Pump-shutdown counterpart of the commit-time persist gate:
+    /// a graceful stop must not lose committed messages still resident in
+    /// the in-memory journal (mirrors the legacy pump's final flush).
+    #[allow(clippy::future_not_send)]
+    pub async fn flush_partitions(&self)
+    where
+        B: MessageBus,
+    {
+        let partitions = self.plane.partitions();
+        let namespaces: Vec<_> = partitions.namespaces().copied().collect();
+        for namespace in namespaces {
+            let Some(partition) = partitions.get_mut_by_ns(&namespace) else {
+                continue;
+            };
+            if let Err(error) = partition
+                .flush_committed_messages(partitions.config())
+                .await
+            {
+                tracing::warn!(
+                    namespace_raw = namespace.inner(),
+                    %error,
+                    "failed to flush partition journal on shutdown"
+                );
+            }
+        }
+    }
+
     #[allow(clippy::future_not_send)]
     pub async fn tick_metadata(&self)
     where
@@ -1859,6 +1895,36 @@ where
     }
 }
 
+/// Broadcast a `StartView` for the current view, answering a replica that
+/// still heartbeats an older view (see `CommitOutcome::RespondStartView`).
+#[allow(clippy::future_not_send)]
+async fn respond_start_view<B, P, J>(consensus: &VsrConsensus<B, P>)
+where
+    B: MessageBus,
+    P: Pipeline<Entry = consensus::PipelineEntry>,
+    J: JournalHandle,
+    <J as JournalHandle>::Target: Journal<
+            <J as JournalHandle>::Storage,
+            Entry = Message<PrepareHeader>,
+            Header = PrepareHeader,
+        >,
+{
+    tracing::info!(
+        view = consensus.view(),
+        op = consensus.sequencer().current_sequence(),
+        commit = consensus.commit_max(),
+        namespace = consensus.namespace(),
+        "answering stale-view heartbeat with StartView"
+    );
+    let action = VsrAction::SendStartView {
+        view: consensus.view(),
+        op: consensus.sequencer().current_sequence(),
+        commit: consensus.commit_max(),
+        namespace: consensus.namespace(),
+    };
+    dispatch_vsr_actions::<B, P, J>(consensus, None, &[action]).await;
+}
+
 /// Dispatch a list of `VsrAction`s by constructing the appropriate
 /// protocol messages and sending them via the consensus message bus.
 #[allow(
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index c71b2464d..90ea1a280 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -329,6 +329,11 @@ where
                 self.apply_reconcile_ops();
             }
         }
+
+        // Final flush: committed messages still resident in the in-memory
+        // journal must reach segment storage before the process exits, or a
+        // graceful restart recovers consumer offsets ahead of the data.
+        self.flush_partitions().await;
     }
 
     /// Sanity check at pump entry: every Consensus frame routed through