You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/06/19 15:18:10 UTC
(iggy) 02/02: fix(consensus): prevent shard panics under in-process Logout storm
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch data-integrity-vsr
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b46634972902006891f6a0d24dd04d1a05f9d1d8
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Fri Jun 19 16:47:10 2026 +0200
fix(consensus): prevent shard panics under in-process Logout storm
A disconnect storm spawns many concurrent in-process Logout submits
on shard 0, breaking two invariants that held only under serial
submission.
dispatch_prepare_and_await (metadata plane) snapshotted view and
commit_min before on_replicate().await and asserted them unchanged
after. A sibling on_ack legitimately advances commit_min while a
task is parked in the await, and a view change advances the view,
tripping the debug_assert. The snapshots fed only the assert; no
release-mode logic read them. Both conditions are handled
downstream - a view change drops the reply_sender so the receiver
resolves Canceled, and loopback acks are op-routed - so the assert
and snapshots are removed.
A new primary floors its view-change pipeline rebuild at the max
commit across the DoViewChange quorum. The DVC carried commit_min
(locally applied), which can lag commit_max (known committed) by
more than the pipeline depth: on_ack pops the committable prefix at
once then applies it across an await per entry, while concurrent
submits push op ahead. The rebuild range then exceeds
PIPELINE_PREPARE_QUEUE_MAX and the assert, or the pipeline push,
panics the shard in release builds. Carry commit_max instead: every
replica holds op - commit_max <= pipeline depth, so the rebuilt
range stays within capacity, while quorum intersection keeps the
floor at or below the winner's op. The committed-but-unapplied tail
is replayed by CommitJournal. Only the value written to the
DoViewChange commit field changes; the wire layout is untouched.
---
core/consensus/src/impls.rs | 26 +++++++++++----------
core/consensus/src/plane_helpers.rs | 46 ++++++++++++++++++++++++++++++++++++-
core/metadata/src/impls/metadata.rs | 15 ++++--------
core/shard/src/lib.rs | 6 ++---
4 files changed, 66 insertions(+), 27 deletions(-)
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index bc6fec07b..5171b4aa1 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -1069,7 +1069,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// Returns a list of actions to take based on fired timeouts.
/// Empty vec means no actions needed.
- pub fn tick(&self, plane: PlaneKind, current_op: u64, current_commit: u64) -> Vec<VsrAction> {
+ pub fn tick(&self, plane: PlaneKind, current_op: u64) -> Vec<VsrAction> {
let mut actions = Vec::new();
let mut timeouts = self.timeouts.borrow_mut();
@@ -1091,11 +1091,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
if timeouts.fired(TimeoutKind::DoViewChangeMessage) {
drop(timeouts);
- actions.extend(self.handle_do_view_change_message_timeout(
- plane,
- current_op,
- current_commit,
- ));
+ actions.extend(self.handle_do_view_change_message_timeout(plane, current_op));
timeouts = self.timeouts.borrow_mut();
}
@@ -1206,7 +1202,6 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
&self,
plane: PlaneKind,
current_op: u64,
- current_commit: u64,
) -> Vec<VsrAction> {
if self.status.get() != Status::ViewChange {
return Vec::new();
@@ -1230,7 +1225,8 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
target: self.primary_index(self.view.get()),
log_view: self.log_view.get(),
op: current_op,
- commit: current_commit,
+ // commit_max, not commit_min: see `handle_start_view_change`.
+ commit: self.commit_max.get(),
namespace: self.namespace,
};
emit_sim_event(
@@ -1471,8 +1467,14 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
let primary_candidate = self.primary_index(self.view.get());
let current_op = self.sequencer.current_sequence();
- // DVC uses commit_min: the replica's actual execution progress.
- let current_commit = self.commit_min.get();
+ // DVC carries commit_max (highest known-committed), not commit_min
+ // (locally applied). The new primary floors its pipeline rebuild at
+ // max(commit) across the quorum; only commit_max bounds that range
+ // to pipeline depth (every replica holds op - commit_max <= depth).
+ // commit_min can lag far behind and overflow the rebuild. The
+ // committed-but-unapplied tail (commit_min..commit_max] is replayed
+ // by the new primary's CommitJournal, not the pipeline.
+ let current_commit = self.commit_max.get();
// Start DVC timeout
self.timeouts
@@ -1605,8 +1607,8 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
}
let current_op = self.sequencer.current_sequence();
- // Use commit_min: the replica's actual execution progress.
- let current_commit = self.commit_min.get();
+ // commit_max, not commit_min: see `handle_start_view_change`.
+ let current_commit = self.commit_max.get();
// If we haven't sent our own DVC yet, record it
if !self.sent_own_do_view_change.get() {
diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs
index e261b79ae..c00aa063c 100644
--- a/core/consensus/src/plane_helpers.rs
+++ b/core/consensus/src/plane_helpers.rs
@@ -456,7 +456,8 @@ pub async fn send_prepare_ok<B, P>(
#[cfg(test)]
mod tests {
use super::*;
- use crate::{Consensus, LocalPipeline};
+ use crate::{Consensus, LocalPipeline, VsrAction};
+ use iggy_binary_protocol::StartViewChangeHeader;
use message_bus::SendError;
use server_common::{MESSAGE_ALIGN, iobuf::Frozen};
@@ -536,6 +537,49 @@ mod tests {
assert!(replicate_preflight(&consensus, current.header()).is_ok());
}
+ // A DoViewChange must carry commit_max (highest known-committed), not
+ // commit_min (locally applied). The new primary floors its pipeline rebuild
+ // at the max DVC commit across the quorum, and only commit_max keeps that
+ // range within pipeline depth (every replica holds op - commit_max <= depth).
+ // commit_min can lag commit_max by more than the depth, so flooring on it
+ // would rebuild an over-deep range and panic the shard. Regression marker.
+ #[test]
+ fn do_view_change_carries_commit_max_not_commit_min() {
+ let consensus = VsrConsensus::new(1, 1, 3, 0, NoopBus, LocalPipeline::new());
+ consensus.init();
+
+ // Diverge the frontiers: applied (commit_min=5) lags known-committed
+ // (commit_max=13) by more than PIPELINE_PREPARE_QUEUE_MAX (8).
+ consensus.advance_commit_max(13);
+ for op in 1..=5 {
+ consensus.advance_commit_min(op);
+ }
+ assert_eq!(consensus.commit_min(), 5);
+ assert_eq!(consensus.commit_max(), 13);
+
+ // An SVC for a higher view from another replica moves this node into the
+ // view change; with f = 1 SVC excluding self, it emits its own DVC.
+ let svc =
+ Message::<StartViewChangeHeader>::new(std::mem::size_of::<StartViewChangeHeader>())
+ .transmute_header(|_, new: &mut StartViewChangeHeader| {
+ new.command = Command2::StartViewChange;
+ new.view = 1;
+ new.replica = 0;
+ new.namespace = 0;
+ });
+ let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header());
+
+ let dvc_commit = actions.iter().find_map(|action| match action {
+ VsrAction::SendDoViewChange { commit, .. } => Some(*commit),
+ _ => None,
+ });
+ assert_eq!(
+ dvc_commit,
+ Some(13),
+ "DVC must carry commit_max (13), not commit_min (5)"
+ );
+ }
+
#[test]
fn loopback_push_and_drain() {
let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new());
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 1fc2d994d..bbac9d2d7 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -1267,12 +1267,6 @@ where
prepare: Message<PrepareHeader>,
) -> Result<Message<ReplyHeader>, Canceled> {
consensus.verify_pipeline();
- // Snapshot (view, commit_min) pre-subscribe. Validate it after
- // `on_replicate` returns: another task could mutate either during the
- // await and silently invalidate the catch-up gate, with release builds
- // proceeding on a stale view-state.
- let view_snapshot = consensus.view();
- let commit_min_snapshot = consensus.commit_min();
let receiver = consensus.pipeline_message_with_subscriber(PlaneKind::Metadata, &prepare);
// Re-check gate post-subscribe: `pipeline_message_with_subscriber`
// can drop the borrow. No commit-max advance flips the gate today;
@@ -1281,11 +1275,12 @@ where
is_caught_up_primary(consensus),
"dispatch_prepare_and_await: gate flipped between check and dispatch"
);
+ // `on_replicate` awaits: a sibling in-process submit may commit
+ // (commit_min advances) or a view change may land (view advances) while
+ // parked here. Both are handled downstream - a view change drops the
+ // reply_sender so `receiver` resolves `Canceled`, and loopback acks are
+ // op-routed - so no post-await view/commit invariant holds or is needed.
self.on_replicate(prepare).await;
- debug_assert!(
- consensus.view() == view_snapshot && consensus.commit_min() == commit_min_snapshot,
- "dispatch_prepare_and_await: view/commit_min advanced across on_replicate await"
- );
let mut loopback = Vec::new();
consensus.drain_loopback_into(&mut loopback);
for message in loopback {
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 8dec40089..691f22cca 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -1673,8 +1673,7 @@ where
let consensus = partition.consensus();
let current_op = consensus.sequencer().current_sequence();
- let current_commit = consensus.commit_min();
- let actions = consensus.tick(PlaneKind::Partitions, current_op, current_commit);
+ let actions = consensus.tick(PlaneKind::Partitions, current_op);
dispatch_vsr_actions::<B, _, MJ>(consensus, None, &actions).await;
dispatch_partition_journal_actions(consensus, partition, &actions).await;
}
@@ -1697,8 +1696,7 @@ where
};
let current_op = consensus.sequencer().current_sequence();
- let current_commit = consensus.commit_min();
- let actions = consensus.tick(PlaneKind::Metadata, current_op, current_commit);
+ let actions = consensus.tick(PlaneKind::Metadata, current_op);
dispatch_vsr_actions(consensus, metadata.journal.as_ref(), &actions).await;
}