You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/06/19 15:18:10 UTC

(iggy) 02/02: fix(consensus): prevent shard panics under in-process Logout storm

This is an automated email from the ASF dual-hosted git repository.

hubcio pushed a commit to branch data-integrity-vsr
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit b46634972902006891f6a0d24dd04d1a05f9d1d8
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Fri Jun 19 16:47:10 2026 +0200

    fix(consensus): prevent shard panics under in-process Logout storm
    
    A disconnect storm spawns many concurrent in-process Logout submits
    on shard 0, breaking two invariants that held only under serial
    submission.
    
    dispatch_prepare_and_await (metadata plane) snapshotted view and
    commit_min before on_replicate().await and asserted them unchanged
    after. A sibling on_ack legitimately advances commit_min while a
    task is parked in the await, and a view change advances the view,
    tripping the debug_assert. The snapshots fed only the assert; no
    release-mode logic read them. Both conditions are handled
    downstream - a view change drops the reply_sender so the receiver
    resolves Canceled, and loopback acks are op-routed - so the assert
    and snapshots are removed.
    
    A new primary floors its view-change pipeline rebuild at the max
    commit across the DoViewChange quorum. The DVC carried commit_min
    (locally applied), which can lag commit_max (known committed) by
    more than the pipeline depth: on_ack pops the committable prefix at
    once then applies it across an await per entry, while concurrent
    submits push op ahead. The rebuild range then exceeds
    PIPELINE_PREPARE_QUEUE_MAX and the assert, or the pipeline push,
    panics the shard in release builds. Carry commit_max instead: every
    replica holds op - commit_max <= pipeline depth, so the rebuilt
    range stays within capacity, while quorum intersection keeps the
    floor at or below the winner's op. The committed-but-unapplied tail
    is replayed by CommitJournal. Only the value written to the
    DoViewChange commit field changes; the wire layout is untouched.
---
 core/consensus/src/impls.rs         | 26 +++++++++++----------
 core/consensus/src/plane_helpers.rs | 46 ++++++++++++++++++++++++++++++++++++-
 core/metadata/src/impls/metadata.rs | 15 ++++--------
 core/shard/src/lib.rs               |  6 ++---
 4 files changed, 66 insertions(+), 27 deletions(-)

diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index bc6fec07b..5171b4aa1 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -1069,7 +1069,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
     ///
     /// Returns a list of actions to take based on fired timeouts.
     /// Empty vec means no actions needed.
-    pub fn tick(&self, plane: PlaneKind, current_op: u64, current_commit: u64) -> Vec<VsrAction> {
+    pub fn tick(&self, plane: PlaneKind, current_op: u64) -> Vec<VsrAction> {
         let mut actions = Vec::new();
         let mut timeouts = self.timeouts.borrow_mut();
 
@@ -1091,11 +1091,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
 
         if timeouts.fired(TimeoutKind::DoViewChangeMessage) {
             drop(timeouts);
-            actions.extend(self.handle_do_view_change_message_timeout(
-                plane,
-                current_op,
-                current_commit,
-            ));
+            actions.extend(self.handle_do_view_change_message_timeout(plane, current_op));
             timeouts = self.timeouts.borrow_mut();
         }
 
@@ -1206,7 +1202,6 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
         &self,
         plane: PlaneKind,
         current_op: u64,
-        current_commit: u64,
     ) -> Vec<VsrAction> {
         if self.status.get() != Status::ViewChange {
             return Vec::new();
@@ -1230,7 +1225,8 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
             target: self.primary_index(self.view.get()),
             log_view: self.log_view.get(),
             op: current_op,
-            commit: current_commit,
+            // commit_max, not commit_min: see `handle_start_view_change`.
+            commit: self.commit_max.get(),
             namespace: self.namespace,
         };
         emit_sim_event(
@@ -1471,8 +1467,14 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
 
             let primary_candidate = self.primary_index(self.view.get());
             let current_op = self.sequencer.current_sequence();
-            // DVC uses commit_min: the replica's actual execution progress.
-            let current_commit = self.commit_min.get();
+            // DVC carries commit_max (highest known-committed), not commit_min
+            // (locally applied). The new primary floors its pipeline rebuild at
+            // max(commit) across the quorum; only commit_max bounds that range
+            // to pipeline depth (every replica holds op - commit_max <= depth).
+            // commit_min can lag far behind and overflow the rebuild. The
+            // committed-but-unapplied tail (commit_min..commit_max] is replayed
+            // by the new primary's CommitJournal, not the pipeline.
+            let current_commit = self.commit_max.get();
 
             // Start DVC timeout
             self.timeouts
@@ -1605,8 +1607,8 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
         }
 
         let current_op = self.sequencer.current_sequence();
-        // Use commit_min: the replica's actual execution progress.
-        let current_commit = self.commit_min.get();
+        // commit_max, not commit_min: see `handle_start_view_change`.
+        let current_commit = self.commit_max.get();
 
         // If we haven't sent our own DVC yet, record it
         if !self.sent_own_do_view_change.get() {
diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs
index e261b79ae..c00aa063c 100644
--- a/core/consensus/src/plane_helpers.rs
+++ b/core/consensus/src/plane_helpers.rs
@@ -456,7 +456,8 @@ pub async fn send_prepare_ok<B, P>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{Consensus, LocalPipeline};
+    use crate::{Consensus, LocalPipeline, VsrAction};
+    use iggy_binary_protocol::StartViewChangeHeader;
     use message_bus::SendError;
     use server_common::{MESSAGE_ALIGN, iobuf::Frozen};
 
@@ -536,6 +537,49 @@ mod tests {
         assert!(replicate_preflight(&consensus, current.header()).is_ok());
     }
 
+    // A DoViewChange must carry commit_max (highest known-committed), not
+    // commit_min (locally applied). The new primary floors its pipeline rebuild
+    // at the max DVC commit across the quorum, and only commit_max keeps that
+    // range within pipeline depth (every replica holds op - commit_max <= depth).
+    // commit_min can lag commit_max by more than the depth, so flooring on it
+    // would rebuild an over-deep range and panic the shard. Regression marker.
+    #[test]
+    fn do_view_change_carries_commit_max_not_commit_min() {
+        let consensus = VsrConsensus::new(1, 1, 3, 0, NoopBus, LocalPipeline::new());
+        consensus.init();
+
+        // Diverge the frontiers: applied (commit_min=5) lags known-committed
+        // (commit_max=13) by more than PIPELINE_PREPARE_QUEUE_MAX (8).
+        consensus.advance_commit_max(13);
+        for op in 1..=5 {
+            consensus.advance_commit_min(op);
+        }
+        assert_eq!(consensus.commit_min(), 5);
+        assert_eq!(consensus.commit_max(), 13);
+
+        // An SVC for a higher view from another replica moves this node into the
+        // view change; with f = 1 SVC excluding self, it emits its own DVC.
+        let svc =
+            Message::<StartViewChangeHeader>::new(std::mem::size_of::<StartViewChangeHeader>())
+                .transmute_header(|_, new: &mut StartViewChangeHeader| {
+                    new.command = Command2::StartViewChange;
+                    new.view = 1;
+                    new.replica = 0;
+                    new.namespace = 0;
+                });
+        let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header());
+
+        let dvc_commit = actions.iter().find_map(|action| match action {
+            VsrAction::SendDoViewChange { commit, .. } => Some(*commit),
+            _ => None,
+        });
+        assert_eq!(
+            dvc_commit,
+            Some(13),
+            "DVC must carry commit_max (13), not commit_min (5)"
+        );
+    }
+
     #[test]
     fn loopback_push_and_drain() {
         let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new());
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 1fc2d994d..bbac9d2d7 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -1267,12 +1267,6 @@ where
         prepare: Message<PrepareHeader>,
     ) -> Result<Message<ReplyHeader>, Canceled> {
         consensus.verify_pipeline();
-        // Snapshot (view, commit_min) pre-subscribe. Validate it after
-        // `on_replicate` returns: another task could mutate either during the
-        // await and silently invalidate the catch-up gate, with release builds
-        // proceeding on a stale view-state.
-        let view_snapshot = consensus.view();
-        let commit_min_snapshot = consensus.commit_min();
         let receiver = consensus.pipeline_message_with_subscriber(PlaneKind::Metadata, &prepare);
         // Re-check gate post-subscribe: `pipeline_message_with_subscriber`
         // can drop the borrow. No commit-max advance flips the gate today;
@@ -1281,11 +1275,12 @@ where
             is_caught_up_primary(consensus),
             "dispatch_prepare_and_await: gate flipped between check and dispatch"
         );
+        // `on_replicate` awaits: a sibling in-process submit may commit
+        // (commit_min advances) or a view change may land (view advances) while
+        // parked here. Both are handled downstream - a view change drops the
+        // reply_sender so `receiver` resolves `Canceled`, and loopback acks are
+        // op-routed - so no post-await view/commit invariant holds or is needed.
         self.on_replicate(prepare).await;
-        debug_assert!(
-            consensus.view() == view_snapshot && consensus.commit_min() == commit_min_snapshot,
-            "dispatch_prepare_and_await: view/commit_min advanced across on_replicate await"
-        );
         let mut loopback = Vec::new();
         consensus.drain_loopback_into(&mut loopback);
         for message in loopback {
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 8dec40089..691f22cca 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -1673,8 +1673,7 @@ where
 
             let consensus = partition.consensus();
             let current_op = consensus.sequencer().current_sequence();
-            let current_commit = consensus.commit_min();
-            let actions = consensus.tick(PlaneKind::Partitions, current_op, current_commit);
+            let actions = consensus.tick(PlaneKind::Partitions, current_op);
             dispatch_vsr_actions::<B, _, MJ>(consensus, None, &actions).await;
             dispatch_partition_journal_actions(consensus, partition, &actions).await;
         }
@@ -1697,8 +1696,7 @@ where
         };
 
         let current_op = consensus.sequencer().current_sequence();
-        let current_commit = consensus.commit_min();
-        let actions = consensus.tick(PlaneKind::Metadata, current_op, current_commit);
+        let actions = consensus.tick(PlaneKind::Metadata, current_op);
 
         dispatch_vsr_actions(consensus, metadata.journal.as_ref(), &actions).await;
     }