You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/08/31 13:47:51 UTC
(iggy) 01/01: fix(shard): drive metadata repair and the commit walk from the tick
This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch metadata_repair_and_commit_from_tick
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 1a9fbc3f83289fbf18d36ab1e653a50ad4aa8c27
Author: Grzegorz Koszyk <g....@gmail.com>
AuthorDate: Mon Aug 31 15:47:38 2026 +0200
fix(shard): drive metadata repair and the commit walk from the tick
---
core/metadata/src/impls/metadata.rs | 45 +++-
core/shard/src/lib.rs | 269 ++++++++++++++-----
core/shard/src/metrics.rs | 24 ++
core/simulator/src/lib.rs | 523 +++++++++++++++++++++++++++++++++++-
4 files changed, 787 insertions(+), 74 deletions(-)
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index b7b7f8337..217e03c69 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -769,6 +769,11 @@ pub struct IggyMetadata<C, J, S, M, SB = PingPongSuperblock> {
/// whole snapshot on shard 0's pump, and hands each requester its own
/// multi-MB copy.
transfer_offer_cache: RefCell<Option<Rc<StateTransferOffer>>>,
+ /// Prepares the backup gap check destroyed since the shard last drained
+ /// the count. Replicated traffic has no client to answer and retransmit
+ /// skips ops that already reached quorum, so nothing else records that the
+ /// frame existed. The partition twin is `IggyPartition::prepare_gap_drops`.
+ prepare_gap_drops: Cell<u64>,
}
impl<C, J, S, M, SB> IggyMetadata<C, J, S, M, SB>
@@ -808,11 +813,27 @@ where
commit_notifier: RefCell::new(None),
client_table_frontier: Cell::new(0),
transfer_offer_cache: RefCell::new(None),
+ prepare_gap_drops: Cell::new(0),
}
}
}
impl<C, J, S, M, SB> IggyMetadata<C, J, S, M, SB> {
+ /// Record one prepare destroyed by the backup gap check. Drained by
+ /// `tick_metadata` into `metadata_prepare_gap_drops_total`; the counter is
+ /// the only production signal that the level-triggered repair driver has
+ /// work to do, since the drop itself answers nobody.
+ pub fn note_prepare_gap_drop(&self) {
+ self.prepare_gap_drops
+ .set(self.prepare_gap_drops.get().saturating_add(1));
+ }
+
+ /// Take and clear the gap-drop count.
+ #[must_use = "dropping the count loses the only record those prepares existed"]
+ pub const fn take_prepare_gap_drops(&self) -> u64 {
+ self.prepare_gap_drops.replace(0)
+ }
+
/// Slot capacity of the LIVE client table, i.e. the largest transferred
/// table this replica can absorb.
///
@@ -1219,6 +1240,7 @@ where
sequencer_op = current_op,
"on_replicate: dropping out-of-order prepare (gap)"
);
+ self.note_prepare_gap_drop();
return;
}
} else {
@@ -3535,10 +3557,10 @@ where
let Some(header) = journal.handle().header(op as usize) else {
// Gap-stop: the walk halts at the first missing prepare and
- // resumes once it is refilled. Live drops refill via the
- // primary's prepare retransmit; a replica behind at recovery
- // or after StartView adoption arms a `MetadataRepairSession`
- // (shard) that re-requests the missing window.
+ // resumes once it is refilled -- by the primary's retransmit
+ // while the op lacks quorum, else by a `MetadataRepairSession`
+ // (armed at StartView adoption or by `tick_metadata`'s gap
+ // detector, since retransmit skips quorum-acked ops).
break;
};
let header = *header;
@@ -4172,6 +4194,21 @@ mod tests {
IggyMetadata::new(None, None, None, None, TestMux::default(), None)
}
+ #[test]
+ fn take_prepare_gap_drops_drains_the_count() {
+ let md = peer_metadata();
+ assert_eq!(md.take_prepare_gap_drops(), 0);
+
+ md.note_prepare_gap_drop();
+ md.note_prepare_gap_drop();
+ assert_eq!(md.take_prepare_gap_drops(), 2);
+ assert_eq!(
+ md.take_prepare_gap_drops(),
+ 0,
+ "a second drain must not re-report drops the metrics already counted"
+ );
+ }
+
#[test]
fn commit_notifier_fires_with_received_operation() {
let md = peer_metadata();
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index ad9b401f7..9dd8d8da1 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -1259,6 +1259,14 @@ where
/// repair takes over at install. See [`MetadataTransferSession`].
metadata_transfer: RefCell<Option<MetadataTransferSession>>,
+ /// Consecutive ticks the metadata group has been seen gap-stopped
+ /// (committed ops it cannot walk to, because the op at its commit frontier
+ /// plus one is missing from the WAL). Debounces `tick_metadata`'s
+ /// level-triggered repair arm; the partition twin is
+ /// `IggyPartition::gap_ticks`. One metadata group per node, so shard-level
+ /// state suffices (precedent: [`Self::metadata_transfer_attempts`]).
+ metadata_gap_ticks: Cell<u32>,
+
/// Serving-side cache of state-transfer offers, both planes, keyed by
/// `(namespace, requester replica id)`. Bounded by the replica count times
/// the groups this shard serves; replaced per fresh nonce.
@@ -1591,6 +1599,7 @@ where
shard_park_shedding: Cell::new(false),
metadata_repair: RefCell::new(None),
metadata_transfer: RefCell::new(None),
+ metadata_gap_ticks: Cell::new(0),
state_transfer_offers: RefCell::new(HashMap::new()),
partition_offer_builds: RefCell::new(HashMap::new()),
served_segment_cache: RefCell::new(ServedSegmentCache::default()),
@@ -1921,6 +1930,7 @@ where
shard_park_shedding: Cell::new(false),
metadata_repair: RefCell::new(None),
metadata_transfer: RefCell::new(None),
+ metadata_gap_ticks: Cell::new(0),
state_transfer_offers: RefCell::new(HashMap::new()),
partition_offer_builds: RefCell::new(HashMap::new()),
served_segment_cache: RefCell::new(ServedSegmentCache::default()),
@@ -4080,17 +4090,16 @@ where
// its own WAL (a late joiner missed the ops below the
// primary's active window; the primary only retransmits
// uncommitted ops, never the committed prefix). Without
- // this, such a replica learns it is behind and does
- // nothing about it -- metadata repair is otherwise only
- // rooted at StartView adoption, which a same-view
- // late joiner never sees. Request repair from the
- // primary; if it has checkpointed past the gap the
- // repair floor evicts and the handler above converts to
- // state transfer. Idempotent: `maybe_request_metadata_repair`
- // no-ops when caught up, already transferring, or a
- // session is live, so a caught-up replica and a
- // cold-start node (commit_max == commit_min == 0) both
- // skip it.
+ // this, such a replica waits out `tick_metadata`'s
+ // debounced gap detector; this edge is the fast path
+ // when a heartbeat does land as `Advanced`. Request
+ // repair from the primary; if it has checkpointed past
+ // the gap the repair floor evicts and the handler above
+ // converts to state transfer. Idempotent:
+ // `maybe_request_metadata_repair` no-ops when caught
+ // up, already transferring, or a session is live, so a
+ // caught-up replica and a cold-start node
+ // (commit_max == commit_min == 0) both skip it.
self.maybe_request_metadata_repair(consensus, header.replica)
.await;
}
@@ -5130,6 +5139,18 @@ where
consensus.group(),
)
.await;
+ } else {
+ // The window is fully walked; only the `RepairDone` that clears
+ // the session was lost, or raced a walk that finished without
+ // it. Nothing is left to request, so close the session here or
+ // it blocks every future arm forever.
+ tracing::info!(
+ shard = self.id,
+ to_op,
+ peer,
+ "metadata repair window fully walked; closing the stalled session"
+ );
+ *self.metadata_repair.borrow_mut() = None;
}
}
}
@@ -5403,8 +5424,10 @@ where
}
/// Start metadata tail journal-repair from `peer` when the commit walk
- /// gap-stopped below the known frontier. Shared by `StartView` adoption
- /// and the post-install step of a state transfer.
+ /// gap-stopped below the known frontier. Every arming site funnels through
+ /// here: `StartView` adoption, the commit-heartbeat backstop, the
+ /// state-transfer fallbacks, and `tick_metadata`'s gap detector, whose
+ /// idempotence these guards provide.
#[allow(clippy::future_not_send)]
async fn maybe_request_metadata_repair<P>(&self, consensus: &VsrConsensus<B, P>, peer: u8)
where
@@ -6645,13 +6668,13 @@ where
self.metrics.record_partition_prepare_gap_drops(gap_drops);
}
let probe = partition_gap_probe(partition);
- let gap_arm = drive_partition_gap_debounce(
+ let gap_arm = drive_gap_debounce(
&probe,
&mut partition.gap_ticks,
repair_retry_ticks,
repair_arms,
);
- (gap_arm, partition_is_walk_stalled(&probe))
+ (gap_arm, is_walk_stalled(&probe))
};
if gap_arm {
let Some(partition) = partitions.get_mut_by_ns(&namespace) else {
@@ -8500,7 +8523,36 @@ where
}
}
- #[allow(clippy::future_not_send)]
+ /// Read the gap probe off the metadata plane; `partition_gap_probe`'s twin.
+ /// A shard method because the recovery slots live here, not on the plane.
+ fn metadata_gap_probe<P>(&self, consensus: &VsrConsensus<B, P>, journal: &MJ) -> GapProbe
+ where
+ B: MessageBus,
+ P: Pipeline<Entry = consensus::PipelineEntry>,
+ MJ: JournalHandle,
+ <MJ as JournalHandle>::Target:
+ Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>,
+ {
+ let commit_min = consensus.commit_min();
+ GapProbe {
+ normal: consensus.is_normal(),
+ transferring: consensus.is_transferring(),
+ recovery: metadata_recovery_owner(
+ self.metadata_transfer.borrow().is_some(),
+ consensus.is_transferring(),
+ self.metadata_repair.borrow().is_some(),
+ ),
+ commit_min,
+ commit_max: consensus.commit_max(),
+ // The same query the commit walk gap-stops on. Safe against the
+ // snapshot floor: a checkpoint drains only to `commit_min`, so
+ // `commit_min + 1` never sits below it and a `None` is a real hole.
+ #[allow(clippy::cast_possible_truncation)]
+ next_op_resident: journal.handle().header((commit_min + 1) as usize).is_some(),
+ }
+ }
+
+ #[allow(clippy::future_not_send, clippy::too_many_lines)]
pub async fn tick_metadata(&self)
where
B: MessageBus,
@@ -8565,6 +8617,56 @@ where
self.advance_pending_metadata_view().await;
self.expire_idle_state_transfer_offers();
+ // Level-triggered gap detector, the metadata twin of the one in
+ // `tick_partitions`; the starvation is the same (`replicate_preflight`
+ // advances `commit_max` before the gap check drops the prepare, so the
+ // `Advanced`-gated arm in `on_commit` never fires under sustained
+ // traffic). Before the transfer-stall block: its exhausted branch
+ // returns early and would skip a detector placed after it.
+ if let Some(journal) = metadata.journal.as_ref() {
+ let gap_drops = metadata.take_prepare_gap_drops();
+ if gap_drops > 0 {
+ self.metrics.record_metadata_prepare_gap_drops(gap_drops);
+ }
+ let probe = self.metadata_gap_probe(consensus, journal);
+ let mut gap_ticks = self.metadata_gap_ticks.get();
+ // Zero arms used: one metadata group per node, no per-tick cap.
+ let gap_arm =
+ drive_gap_debounce(&probe, &mut gap_ticks, self.repair_retry_ticks.get(), 0);
+ self.metadata_gap_ticks.set(gap_ticks);
+ if gap_arm {
+ let peer = consensus.primary_index(consensus.view());
+ tracing::info!(
+ shard = self.id,
+ commit_min = probe.commit_min,
+ commit_max = probe.commit_max,
+ peer,
+ "metadata gap-stopped past the debounce; arming repair from the primary"
+ );
+ // Always repair, never classify the gap up front: a window
+ // below the peer's retention floor is answered `RangeEvicted`,
+ // and `on_repair_range_reply` converts that to a state
+ // transfer. The floor is only learned through that refusal.
+ self.maybe_request_metadata_repair(consensus, peer).await;
+ }
+ // Undebounced and uncapped, like the partition walk arm. Follower
+ // only: a backup's `commit_journal` ships no wire replies, while a
+ // stranded primary is `resume_stranded_commits`' job above, which
+ // does. Not gated on `RecoveryOwner` (repaired prepares are
+ // journaled without walking, so gating parks the walk all
+ // session); `is_walk_stalled` itself refuses mid-transfer, where a
+ // walk past the incoming `snapshot_seq` breaks the install.
+ if is_walk_stalled(&probe) && consensus.is_follower() {
+ tracing::info!(
+ shard = self.id,
+ commit_min = probe.commit_min,
+ commit_max = probe.commit_max,
+ "metadata commit walk parked over resident committed ops; resuming"
+ );
+ metadata.commit_journal().await;
+ }
+ }
+
// Stall retry for an in-flight state transfer: descriptor or chunk
// frames are fire-and-forget, so a lost one must not wedge the
// session (and the boot flow behind it) forever.
@@ -8907,19 +9009,20 @@ fn repair_serve_ceiling(requested_to_op: u64, commit_max: u64, head: u64) -> u64
/// tick. Over-cap groups stay due and arm on a later pass.
const PARTITION_REPAIR_ARMS_PER_TICK_MAX: usize = 3;
-/// What already owns a partition's recovery, if anything.
+/// What already owns a group's recovery, if anything.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RecoveryOwner {
- /// Nothing owns it: the sweep may arm repair.
+ /// Nothing owns it: the tick may arm repair.
Nobody,
Repair,
Transfer,
/// A scheduled transfer re-arm counting down. Arming repair over it would
- /// defeat its backoff, as `arm_partition_transfer` documents.
+ /// defeat its backoff, as `arm_partition_transfer` documents. Partition
+ /// plane only; metadata has no re-arm state.
TransferRearm,
}
-/// What the tick sweep reads off one partition to decide whether it is
+/// What a tick driver reads off one consensus group to decide whether it is
/// gap-stopped. Split out so the guards, the debounce and the per-tick cap are
/// testable without a shard, a bus, or a journal.
#[derive(Debug, Clone, Copy)]
@@ -8939,8 +9042,8 @@ struct GapProbe {
/// The journal-hole half is not redundant: a follower advances `commit_max`
/// from every prepare header in `replicate_preflight`, so `commit_min <
/// commit_max` is transiently true on every healthy pipelined tick and a bare
-/// lag test would arm repair against ordinary produce.
-const fn partition_is_gap_stopped(probe: &GapProbe) -> bool {
+/// lag test would arm repair against ordinary traffic.
+const fn is_gap_stopped(probe: &GapProbe) -> bool {
probe.normal
&& !probe.transferring
&& matches!(probe.recovery, RecoveryOwner::Nobody)
@@ -8952,31 +9055,32 @@ const fn partition_is_gap_stopped(probe: &GapProbe) -> bool {
/// just never ran (a heartbeat carrying a known commit is `Accepted`, and an
/// idle group offers no other edge). Not gated on `RecoveryOwner`: repair
/// fetches bodies without walking them, so gating parks the walk all session.
-const fn partition_is_walk_stalled(probe: &GapProbe) -> bool {
+const fn is_walk_stalled(probe: &GapProbe) -> bool {
probe.normal
&& !probe.transferring
&& probe.commit_min < probe.commit_max
&& probe.next_op_resident
}
-/// Count one sweep tick against `gap_ticks` and answer whether this partition
-/// may arm repair now.
+/// Count one tick against `gap_ticks` and answer whether this group may arm
+/// repair now.
///
/// Level-triggered, because every edge-triggered arming site is starvable: the
/// commit-heartbeat backstop fires only on `CommitOutcome::Advanced`, and under
-/// sustained produce the prepares consume the advance in preflight before the
+/// sustained traffic the prepares consume the advance in preflight before the
/// gap check drops them, so the heartbeat lands as `Accepted` and the gap wedges
/// until an unrelated view change.
///
/// A capped-out arm keeps its debounce satisfied rather than starting over, so
-/// the group arms on the next pass with a slot free.
-const fn drive_partition_gap_debounce(
+/// the group arms on the next pass with a slot free. The cap only bounds the
+/// partition sweep; the metadata driver has one group and passes zero arms.
+const fn drive_gap_debounce(
probe: &GapProbe,
gap_ticks: &mut u32,
debounce_ticks: u32,
arms_this_tick: usize,
) -> bool {
- if !partition_is_gap_stopped(probe) {
+ if !is_gap_stopped(probe) {
*gap_ticks = 0;
return false;
}
@@ -9018,6 +9122,25 @@ where
}
}
+/// Map the metadata plane's recovery slots onto [`RecoveryOwner`]. Transfer
+/// first, as in `partition_gap_probe`: it supersedes repair, so naming it is
+/// the truthful diagnostic when both are set. The stage counts as a transfer
+/// too, since `begin_state_transfer_await` owns the recovery before a session
+/// exists.
+const fn metadata_recovery_owner(
+ transfer_session: bool,
+ stage_transferring: bool,
+ repair_session: bool,
+) -> RecoveryOwner {
+ if transfer_session || stage_transferring {
+ RecoveryOwner::Transfer
+ } else if repair_session {
+ RecoveryOwner::Repair
+ } else {
+ RecoveryOwner::Nobody
+ }
+}
+
/// Whether the parked `StartView` log names every op in the uncommitted
/// suffix `(commit_max, head]`, in descending order. Only this canonical list
/// makes fetching bodies above the commit point safe.
@@ -10254,15 +10377,16 @@ mod superblock_fail_stop_tests {
#[cfg(test)]
mod gap_detector_tests {
- //! The level-triggered repair arm the partition tick sweep runs.
+ //! The level-triggered repair arm the partition and metadata tick drivers
+ //! share.
//!
//! Its whole reason to exist is that the edge-triggered arming sites are
//! starvable, so the guards it shares with them and the debounce that keeps
//! it off healthy traffic are the parts worth pinning.
use super::{
- GapProbe, PARTITION_REPAIR_ARMS_PER_TICK_MAX, RecoveryOwner, drive_partition_gap_debounce,
- partition_is_gap_stopped, partition_is_walk_stalled,
+ GapProbe, PARTITION_REPAIR_ARMS_PER_TICK_MAX, RecoveryOwner, drive_gap_debounce,
+ is_gap_stopped, is_walk_stalled, metadata_recovery_owner,
};
const DEBOUNCE: u32 = 100;
@@ -10300,8 +10424,8 @@ mod gap_detector_tests {
next_op_resident: true,
..gap_stopped()
};
- assert!(!partition_is_gap_stopped(&healthy));
- assert!(partition_is_gap_stopped(&gap_stopped()));
+ assert!(!is_gap_stopped(&healthy));
+ assert!(is_gap_stopped(&gap_stopped()));
}
#[test]
@@ -10310,7 +10434,7 @@ mod gap_detector_tests {
commit_min: 10,
..gap_stopped()
};
- assert!(!partition_is_gap_stopped(&caught_up));
+ assert!(!is_gap_stopped(&caught_up));
}
#[test]
@@ -10321,13 +10445,13 @@ mod gap_detector_tests {
normal: false,
..gap_stopped()
};
- assert!(!partition_is_gap_stopped(&electing));
+ assert!(!is_gap_stopped(&electing));
let installing = GapProbe {
transferring: true,
..gap_stopped()
};
- assert!(!partition_is_gap_stopped(&installing));
+ assert!(!is_gap_stopped(&installing));
}
#[test]
@@ -10342,7 +10466,7 @@ mod gap_detector_tests {
..gap_stopped()
};
assert!(
- !partition_is_gap_stopped(&owned),
+ !is_gap_stopped(&owned),
"{owner:?} owns the recovery; a second session would race it"
);
}
@@ -10354,16 +10478,11 @@ mod gap_detector_tests {
let mut gap_ticks = 0;
for tick in 1..DEBOUNCE {
assert!(
- !drive_partition_gap_debounce(&probe, &mut gap_ticks, DEBOUNCE, 0),
+ !drive_gap_debounce(&probe, &mut gap_ticks, DEBOUNCE, 0),
"armed at tick {tick}, before the debounce elapsed"
);
}
- assert!(drive_partition_gap_debounce(
- &probe,
- &mut gap_ticks,
- DEBOUNCE,
- 0
- ));
+ assert!(drive_gap_debounce(&probe, &mut gap_ticks, DEBOUNCE, 0));
}
#[test]
@@ -10375,28 +10494,23 @@ mod gap_detector_tests {
};
let mut gap_ticks = 0;
for _ in 0..DEBOUNCE - 1 {
- drive_partition_gap_debounce(&stopped, &mut gap_ticks, DEBOUNCE, 0);
+ drive_gap_debounce(&stopped, &mut gap_ticks, DEBOUNCE, 0);
}
assert_eq!(gap_ticks, DEBOUNCE - 1);
- assert!(!drive_partition_gap_debounce(
- &walkable,
- &mut gap_ticks,
- DEBOUNCE,
- 0
- ));
+ assert!(!drive_gap_debounce(&walkable, &mut gap_ticks, DEBOUNCE, 0));
assert_eq!(gap_ticks, 0, "progress must restart the debounce");
assert!(
- !drive_partition_gap_debounce(&stopped, &mut gap_ticks, DEBOUNCE, 0),
+ !drive_gap_debounce(&stopped, &mut gap_ticks, DEBOUNCE, 0),
"a fresh gap must serve its own debounce, not inherit the old count"
);
}
#[test]
fn given_a_follower_with_resident_committed_ops_when_probed_should_be_walk_stalled() {
- assert!(partition_is_walk_stalled(&walk_stalled()));
+ assert!(is_walk_stalled(&walk_stalled()));
assert!(
- !partition_is_walk_stalled(&gap_stopped()),
+ !is_walk_stalled(&gap_stopped()),
"a missing next op is repair's job; a walk over it would stop dead"
);
}
@@ -10407,7 +10521,7 @@ mod gap_detector_tests {
commit_min: 10,
..walk_stalled()
};
- assert!(!partition_is_walk_stalled(&caught_up));
+ assert!(!is_walk_stalled(&caught_up));
}
#[test]
@@ -10416,7 +10530,7 @@ mod gap_detector_tests {
normal: false,
..walk_stalled()
};
- assert!(!partition_is_walk_stalled(&electing));
+ assert!(!is_walk_stalled(&electing));
// Same gate as the on-commit arm: a walk during a transfer can advance
// commit_min past the incoming frontier.
@@ -10424,7 +10538,7 @@ mod gap_detector_tests {
transferring: true,
..walk_stalled()
};
- assert!(!partition_is_walk_stalled(&installing));
+ assert!(!is_walk_stalled(&installing));
}
#[test]
@@ -10442,7 +10556,7 @@ mod gap_detector_tests {
..walk_stalled()
};
assert!(
- partition_is_walk_stalled(&owned),
+ is_walk_stalled(&owned),
"{owner:?} owns the fetch, not the resident prefix"
);
}
@@ -10472,8 +10586,7 @@ mod gap_detector_tests {
next_op_resident,
};
assert!(
- !(partition_is_gap_stopped(&probe)
- && partition_is_walk_stalled(&probe)),
+ !(is_gap_stopped(&probe) && is_walk_stalled(&probe)),
"both predicates claim {probe:?}"
);
}
@@ -10488,7 +10601,7 @@ mod gap_detector_tests {
let probe = gap_stopped();
let mut gap_ticks = DEBOUNCE;
assert!(
- !drive_partition_gap_debounce(
+ !drive_gap_debounce(
&probe,
&mut gap_ticks,
DEBOUNCE,
@@ -10502,7 +10615,7 @@ mod gap_detector_tests {
arm a whole interval out per contended tick"
);
assert!(
- drive_partition_gap_debounce(
+ drive_gap_debounce(
&probe,
&mut gap_ticks,
DEBOUNCE,
@@ -10511,4 +10624,34 @@ mod gap_detector_tests {
"the same group arms on the next pass with a slot free"
);
}
+
+ #[test]
+ fn given_metadata_recovery_slots_when_mapped_should_name_transfer_over_repair() {
+ // (transfer session, stage transferring, repair session), as
+ // `metadata_gap_probe` reads them off `metadata_transfer`, the
+ // consensus stage and `metadata_repair`.
+ assert_eq!(
+ metadata_recovery_owner(false, false, false),
+ RecoveryOwner::Nobody
+ );
+ assert_eq!(
+ metadata_recovery_owner(false, false, true),
+ RecoveryOwner::Repair
+ );
+ // A stage past `Idle` owns the recovery before any session exists.
+ assert_eq!(
+ metadata_recovery_owner(false, true, false),
+ RecoveryOwner::Transfer
+ );
+ assert_eq!(
+ metadata_recovery_owner(true, false, false),
+ RecoveryOwner::Transfer
+ );
+ assert_eq!(
+ metadata_recovery_owner(true, true, true),
+ RecoveryOwner::Transfer,
+ "a transfer supersedes a lingering repair session; naming Repair \
+ would read as the tick may re-arm it"
+ );
+ }
}
diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs
index cefc3d445..29f0a4729 100644
--- a/core/shard/src/metrics.rs
+++ b/core/shard/src/metrics.rs
@@ -200,6 +200,7 @@ pub struct ShardMetrics {
partition_requests_denied_transient_total: Counter,
partition_repair_serves_deferred_purge_total: Counter,
partition_prepare_gap_drops_total: Counter,
+ metadata_prepare_gap_drops_total: Counter,
}
impl ShardMetrics {
@@ -225,6 +226,7 @@ impl ShardMetrics {
partition_requests_denied_transient_total: Counter::default(),
partition_repair_serves_deferred_purge_total: Counter::default(),
partition_prepare_gap_drops_total: Counter::default(),
+ metadata_prepare_gap_drops_total: Counter::default(),
}
}
@@ -417,6 +419,23 @@ impl ShardMetrics {
self.partition_prepare_gap_drops_total.get()
}
+ /// Add the prepares the metadata backup gap check destroyed since the last
+ /// tick, drained from `IggyMetadata::take_prepare_gap_drops`. A sibling of
+ /// `partition_prepare_gap_drops_total`, and NOT a `frame_drops_total`
+ /// reason for the same cause: gap drops are protocol-ordering drops that
+ /// the tick driver repairs, not routing faults, and the simulator asserts
+ /// `frame_drops_total` stays at zero on runs with no injected loss.
+ pub fn record_metadata_prepare_gap_drops(&self, drops: u64) {
+ self.metadata_prepare_gap_drops_total.inc_by(drops);
+ }
+
+ /// Snapshot of `metadata_prepare_gap_drops_total`. Test/simulator accessor.
+ #[cfg(any(test, feature = "simulator"))]
+ #[must_use]
+ pub fn metadata_prepare_gap_drops_value(&self) -> u64 {
+ self.metadata_prepare_gap_drops_total.get()
+ }
+
/// Snapshot of `partition_frames_rejected_stale_total`. Test/simulator
/// accessor, readable from any crate under those cfgs so the crates that
/// drive the reconciler can assert a reject did not happen.
@@ -513,6 +532,11 @@ impl ShardMetrics {
"replicated prepares dropped out of order by a backup's gap check",
self.partition_prepare_gap_drops_total.clone(),
);
+ registry.register(
+ "metadata_prepare_gap_drops",
+ "replicated metadata prepares dropped out of order by a backup's gap check",
+ self.metadata_prepare_gap_drops_total.clone(),
+ );
}
}
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 44421de46..44abeb498 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -4740,9 +4740,10 @@ mod partition_repair_driver_tests {
/// the healer.
const STRAND_QUIET_STEPS: usize = 300;
- /// The partition-plane prepare a packet carries, if it carries one for
- /// `group`.
- fn prepare_for(packet: &Packet, group: u64) -> Option<PrepareHeader> {
+ /// The prepare a packet carries, if it carries one for `group`. Shared
+ /// with `metadata_repair_driver_tests`, as are the three below and
+ /// [`cluster`]: both planes ride the same commands, keyed by group.
+ pub fn prepare_for(packet: &Packet, group: u64) -> Option<PrepareHeader> {
if packet.message.header().command != Command::Prepare {
return None;
}
@@ -4752,7 +4753,7 @@ mod partition_repair_driver_tests {
}
/// Whether a packet is a commit heartbeat for `group`.
- fn is_commit_for(packet: &Packet, group: u64) -> bool {
+ pub fn is_commit_for(packet: &Packet, group: u64) -> bool {
if packet.message.header().command != Command::Commit {
return false;
}
@@ -4762,7 +4763,7 @@ mod partition_repair_driver_tests {
}
/// Whether a packet is a repair request for `group`.
- fn is_request_prepares_for(packet: &Packet, group: u64) -> bool {
+ pub fn is_request_prepares_for(packet: &Packet, group: u64) -> bool {
if packet.message.header().command != Command::RequestPrepares {
return false;
}
@@ -4773,7 +4774,7 @@ mod partition_repair_driver_tests {
}
/// Whether a packet is a repair stream terminator for `group`.
- fn is_repair_done_for(packet: &Packet, group: u64) -> bool {
+ pub fn is_repair_done_for(packet: &Packet, group: u64) -> bool {
if packet.message.header().command != Command::RepairDone {
return false;
}
@@ -4783,7 +4784,7 @@ mod partition_repair_driver_tests {
header.group == group
}
- fn cluster(seed: u64) -> (Simulator, SimClient) {
+ pub fn cluster(seed: u64) -> (Simulator, SimClient) {
server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings {
enabled: false,
size: iggy_common::IggyByteSize::from(0u64),
@@ -5300,3 +5301,511 @@ mod partition_repair_driver_tests {
);
}
}
+
+#[cfg(test)]
+mod metadata_repair_driver_tests {
+ //! A backup that missed a committed metadata prepare recovers in Normal
+ //! status, without waiting for a view change.
+ //!
+ //! The metadata twin of `partition_repair_driver_tests`, driven by the
+ //! detector in `tick_metadata`: the same preflight `commit_max` advance
+ //! starves the `Advanced`-gated arm in `on_commit`, and
+ //! `retry_stalled_metadata_repair` re-drives only a session that already
+ //! exists. Every fault here is keyed on `METADATA_GROUP`, since both
+ //! planes share `Prepare`/`Commit` on the same links.
+
+ use super::partition_repair_driver_tests::{
+ cluster, is_commit_for, is_repair_done_for, is_request_prepares_for, prepare_for,
+ };
+ use super::*;
+ use consensus::Status;
+ use journal::Journal;
+ use packet::Packet;
+ use server_common::sharding::METADATA_GROUP;
+ use std::sync::atomic::{AtomicU64, Ordering};
+
+ /// Chain replication runs 0 -> 1 -> 2 and stops before the primary, so
+ /// replica 2 is the only one whose losses cannot starve the group of
+ /// quorum (see `partition_repair_driver_tests::LAGGING`).
+ const LAGGING: u8 = 2;
+
+ const CLIENT_ID: u128 = 1;
+
+ /// Ops committed cleanly before the fault, so the gap opens above a
+ /// committed prefix rather than at the group's first op.
+ const WARMUP_SENDS: usize = 3;
+
+ /// Ticks stepped after each stream creation, one round trip's worth.
+ const STEPS_PER_SEND: usize = 12;
+
+ /// Creations issued with the fault standing in the gap test. Long enough
+ /// that prepares keep consuming the `commit_max` advance the heartbeat
+ /// backstop needs; the debounce may elapse mid-produce, which the verdict
+ /// tolerates (the withheld heartbeats mean only the tick driver can arm).
+ const GAP_SENDS: usize = 12;
+
+ /// Creations issued with the fault standing in the eviction and
+ /// walk-starvation tests: few enough (under `partitions::REPAIR_RETRY_TICKS`
+ /// ticks) that the debounce fires only after produce stops, so the floor
+ /// stamp / the starved walk edge is in place before the arm runs.
+ const SHORT_GAP_SENDS: usize = 3;
+
+ /// Quiet ticks for the repair stream to land, kept under
+ /// `NORMAL_HEARTBEAT_TICKS` (500) so no election can be the healer.
+ const QUIET_STEPS: usize = 160;
+
+ /// Budget for the group to settle once the fault is lifted; the drain
+ /// loop breaks on convergence.
+ const DRAIN_STEPS: usize = 600;
+
+ /// Quiet budget for the walk-starvation test: debounce, repair stream,
+ /// then the drain, still under `NORMAL_HEARTBEAT_TICKS`.
+ const STRAND_QUIET_STEPS: usize = 300;
+
+ /// Ticks of healthy load in the no-false-positive test, several debounce
+ /// intervals' worth so the driver gets many chances to arm.
+ const LOAD_TICKS: usize = 4 * partitions::REPAIR_RETRY_TICKS as usize;
+
+ /// Paced at a fraction of a round trip; faster submission only collects
+ /// transient rejections once the prepare pipeline fills.
+ const TICKS_PER_SEND: usize = 4;
+
+ /// Ops the healthy run must have committed for its verdict to mean
+ /// anything: enough to prove the group was live across several debounce
+ /// intervals, not that it was saturated.
+ const COMMITTED_MIN: u64 = 20;
+
+ /// `(status, view, commit_min, commit_max)` of one replica's metadata group.
+ fn metadata_state(sim: &Simulator, replica: u8) -> (Status, u32, u64, u64) {
+ let metadata = sim.replicas[replica as usize].shards[0].plane.metadata();
+ let consensus = metadata
+ .consensus
+ .as_ref()
+ .expect("shard 0 owns metadata consensus");
+ (
+ consensus.status(),
+ consensus.view(),
+ consensus.commit_min(),
+ consensus.commit_max(),
+ )
+ }
+
+ #[allow(clippy::cast_possible_truncation)]
+ fn journal_holds(sim: &Simulator, replica: u8, op: u64) -> bool {
+ sim.replicas[replica as usize]
+ .metadata_journal
+ .header(op as usize)
+ .is_some()
+ }
+
+ fn gap_drops(sim: &Simulator, replica: u8) -> u64 {
+ sim.replicas[replica as usize].shards[0]
+ .metrics()
+ .metadata_prepare_gap_drops_value()
+ }
+
+ fn transfer_armed(sim: &Simulator, replica: u8) -> bool {
+ let metadata = sim.replicas[replica as usize].shards[0].plane.metadata();
+ metadata.consensus.as_ref().is_some_and(|consensus| {
+ consensus.state_transfer_stage() != consensus::StateTransferStage::Idle
+ })
+ }
+
+ /// Submit `sends` stream creations to the primary, stepping between each.
+ fn create_streams(sim: &mut Simulator, client: &SimClient, sends: usize, tag: &str) {
+ for index in 0..sends {
+ let msg = client.create_stream(&format!("{tag}-{index}"));
+ sim.submit_request(client.client_id(), 0, msg.into_generic());
+ for _ in 0..STEPS_PER_SEND {
+ sim.step();
+ }
+ }
+ }
+
+ #[test]
+ fn given_a_backup_that_dropped_a_committed_metadata_prepare_when_heartbeat_advances_are_starved_should_repair_in_normal_status()
+ {
+ // Statics, not captures: the link hooks are bare `fn` pointers,
+ // declared inside the test so parallel siblings cannot share them.
+ static WITHHELD_OP: AtomicU64 = AtomicU64::new(0);
+
+ /// Chain link 1 -> 2: swallow the first metadata prepare, once.
+ fn withhold_one_prepare(packet: &Packet) -> bool {
+ let Some(header) = prepare_for(packet, METADATA_GROUP) else {
+ return false;
+ };
+ WITHHELD_OP
+ .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed)
+ .is_ok()
+ }
+
+ /// Primary -> 2: withhold this group's commit heartbeats, so the
+ /// `Advanced` backstop can never run, and withhold retransmits of the
+ /// dropped op. The retransmit half stands in for production behaviour:
+ /// `consensus::retransmit_targets` skips an op that already reached
+ /// quorum, and this op reaches quorum on 0 and 1 alone.
+ fn starve_commit_edge(packet: &Packet) -> bool {
+ if let Some(header) = prepare_for(packet, METADATA_GROUP) {
+ return header.op == WITHHELD_OP.load(Ordering::Relaxed);
+ }
+ is_commit_for(packet, METADATA_GROUP)
+ }
+
+ let (mut sim, client) = cluster(0x5EED_0240);
+ sim.register_client_with_primary(&client);
+ WITHHELD_OP.store(0, Ordering::Relaxed);
+
+ create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm");
+ let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING);
+ assert!(
+ warm_commit_min > 0,
+ "the lagging replica committed nothing before the fault, so the gap \
+ below would open at the group's first op"
+ );
+
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) =
+ Some(withhold_one_prepare);
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) =
+ Some(starve_commit_edge);
+
+ create_streams(&mut sim, &client, GAP_SENDS, "md-gap");
+
+ let withheld = WITHHELD_OP.load(Ordering::Relaxed);
+ assert_ne!(
+ withheld, 0,
+ "no metadata prepare crossed the chain link, so the fault never armed"
+ );
+ assert!(
+ gap_drops(&sim, LAGGING) > 0,
+ "the lagging replica never reached its backup gap check, so the \
+ prepares after the withheld op were not dropped as a gap"
+ );
+
+ for _ in 0..QUIET_STEPS {
+ sim.step();
+ }
+
+ // Judged with the blockade still standing: no commit heartbeat for
+ // this group has reached the replica since the gap opened, so only
+ // the tick driver can have armed the repair.
+ let (status, view, commit_min, _) = metadata_state(&sim, LAGGING);
+ assert_eq!(
+ view, 0,
+ "a view change healed the gap instead of the repair driver; the test \
+ proves nothing about normal status"
+ );
+ assert_eq!(status, Status::Normal, "the replica left Normal status");
+ assert!(
+ journal_holds(&sim, LAGGING, withheld),
+ "op {withheld} was never repaired back into the lagging replica's WAL"
+ );
+ assert!(
+ commit_min >= withheld,
+ "the commit walk never crossed the repaired hole: stopped at \
+ {commit_min}, the withheld op is {withheld}"
+ );
+
+ // Lift the blockade and let the group settle; the tail above the
+ // repaired window waits on the heartbeats the fault withheld.
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = None;
+ for _ in 0..DRAIN_STEPS {
+ sim.step();
+ let (_, _, commit_min, commit_max) = metadata_state(&sim, LAGGING);
+ if commit_min == commit_max {
+ break;
+ }
+ }
+ let (status, view, commit_min, commit_max) = metadata_state(&sim, LAGGING);
+ assert_eq!((status, view), (Status::Normal, 0));
+ assert_eq!(
+ commit_min, commit_max,
+ "the lagging replica is still gap-stopped: committed through \
+ {commit_max} but walkable only to {commit_min}"
+ );
+ }
+
+ #[test]
+ fn given_a_metadata_repair_armed_by_the_tick_driver_when_the_floor_is_evicted_should_convert_to_state_transfer()
+ {
+ static WITHHELD_OP: AtomicU64 = AtomicU64::new(0);
+
+ fn withhold_one_prepare(packet: &Packet) -> bool {
+ let Some(header) = prepare_for(packet, METADATA_GROUP) else {
+ return false;
+ };
+ WITHHELD_OP
+ .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed)
+ .is_ok()
+ }
+
+ fn starve_commit_edge(packet: &Packet) -> bool {
+ if let Some(header) = prepare_for(packet, METADATA_GROUP) {
+ return header.op == WITHHELD_OP.load(Ordering::Relaxed);
+ }
+ is_commit_for(packet, METADATA_GROUP)
+ }
+
+ let (mut sim, client) = cluster(0x5EED_0241);
+ sim.register_client_with_primary(&client);
+ WITHHELD_OP.store(0, Ordering::Relaxed);
+
+ create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm");
+
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) =
+ Some(withhold_one_prepare);
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) =
+ Some(starve_commit_edge);
+
+ create_streams(&mut sim, &client, SHORT_GAP_SENDS, "md-gap");
+ assert_ne!(
+ WITHHELD_OP.load(Ordering::Relaxed),
+ 0,
+ "no metadata prepare crossed the chain link, so the fault never armed"
+ );
+
+ // Move the primary's retention floor past the whole gap window before
+ // the debounce can arm (`SHORT_GAP_SENDS`): the serve path reads only
+ // the snapshot watermark, so the request is answered `RangeEvicted`
+ // (see `stamp_metadata_snapshot`).
+ let primary_commit_min = metadata_state(&sim, 0).2;
+ sim.stamp_metadata_snapshot(0, primary_commit_min);
+
+ for _ in 0..QUIET_STEPS {
+ sim.step();
+ if transfer_armed(&sim, LAGGING) {
+ break;
+ }
+ }
+
+ let (status, view, ..) = metadata_state(&sim, LAGGING);
+ assert_eq!(
+ view, 0,
+ "a view change armed the recovery instead of the tick-armed repair session"
+ );
+ assert!(
+ transfer_armed(&sim, LAGGING),
+ "the tick-armed repair session hit an evicted floor but never converted \
+ to a state transfer (status {status:?})"
+ );
+ }
+
+ #[test]
+ fn given_a_backup_holding_resident_committed_metadata_ops_when_every_walk_edge_is_starved_should_drain_in_normal_status()
+ {
+ static WITHHELD_OP: AtomicU64 = AtomicU64::new(0);
+ static WITHHELD_DONES: AtomicU64 = AtomicU64::new(0);
+
+ /// Chain link 1 -> 2: swallow the first metadata prepare, once.
+ fn withhold_one_prepare(packet: &Packet) -> bool {
+ let Some(header) = prepare_for(packet, METADATA_GROUP) else {
+ return false;
+ };
+ WITHHELD_OP
+ .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed)
+ .is_ok()
+ }
+
+ /// Primary -> 2: withhold every direct prepare (live ones ride the
+ /// chain, so this starves only retransmit heals), the group's commit
+ /// heartbeats, and its repair terminators. The repaired ops themselves
+ /// pass, so the window lands resident while the walk `RepairDone`
+ /// would run never fires.
+ fn starve_walk_edges(packet: &Packet) -> bool {
+ if prepare_for(packet, METADATA_GROUP).is_some() {
+ return true;
+ }
+ if is_repair_done_for(packet, METADATA_GROUP) {
+ WITHHELD_DONES.fetch_add(1, Ordering::Relaxed);
+ return true;
+ }
+ is_commit_for(packet, METADATA_GROUP)
+ }
+
+ let (mut sim, client) = cluster(0x5EED_0242);
+ sim.register_client_with_primary(&client);
+ WITHHELD_OP.store(0, Ordering::Relaxed);
+ WITHHELD_DONES.store(0, Ordering::Relaxed);
+
+ create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm");
+ let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING);
+ assert!(
+ warm_commit_min > 0,
+ "the lagging replica committed nothing before the fault, so the gap \
+ below would open at the group's first op"
+ );
+
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) =
+ Some(withhold_one_prepare);
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) =
+ Some(starve_walk_edges);
+
+ create_streams(&mut sim, &client, SHORT_GAP_SENDS, "md-strand");
+
+ let withheld = WITHHELD_OP.load(Ordering::Relaxed);
+ assert_ne!(
+ withheld, 0,
+ "no metadata prepare crossed the chain link, so the fault never armed"
+ );
+
+ for _ in 0..STRAND_QUIET_STEPS {
+ sim.step();
+ let (_, view, commit_min, commit_max) = metadata_state(&sim, LAGGING);
+ if view != 0 || (commit_min >= withheld && commit_min == commit_max) {
+ break;
+ }
+ }
+
+ assert!(
+ WITHHELD_DONES.load(Ordering::Relaxed) > 0,
+ "no repair terminator was withheld, so the walk was never starved and \
+ a green run would not prove the tick backstop"
+ );
+ assert!(
+ journal_holds(&sim, LAGGING, withheld),
+ "op {withheld} was never repaired back into the lagging replica's WAL"
+ );
+ let (status, view, commit_min, commit_max) = metadata_state(&sim, LAGGING);
+ assert_eq!(
+ view, 0,
+ "a view change drained the walk instead of the tick backstop; the test \
+ proves nothing about normal status"
+ );
+ assert_eq!(status, Status::Normal, "the replica left Normal status");
+ for op in commit_min + 1..=commit_max {
+ assert!(
+ journal_holds(&sim, LAGGING, op),
+ "op {op} is not resident, so this run stranded on a repair gap, \
+ not a parked walk"
+ );
+ }
+ assert_eq!(
+ commit_min, commit_max,
+ "the walk never resumed over resident committed ops: walkable to \
+ {commit_min}, committed through {commit_max}, every op between resident"
+ );
+ }
+
+ /// Regression canary, expected green even without the tick driver: a
+ /// healthy metadata backup walks at every accepted prepare's tail
+ /// (`on_replicate`), so per-tick lag never survives to quiescence and the
+ /// journal-hole half of the predicate is pinned by `gap_detector_tests`
+ /// instead. What this run pins is that the driver stays silent under
+ /// sustained pipelined load.
+ #[test]
+ fn given_healthy_metadata_traffic_when_no_gap_exists_should_not_arm_repair() {
+ static REPAIR_REQUESTS: AtomicU64 = AtomicU64::new(0);
+
+ /// Observer, not a fault: counts this group's repair requests and
+ /// passes every packet through.
+ fn count_repair_requests(packet: &Packet) -> bool {
+ if is_request_prepares_for(packet, METADATA_GROUP) {
+ REPAIR_REQUESTS.fetch_add(1, Ordering::Relaxed);
+ }
+ false
+ }
+
+ // TWO replicas, as in the partition twin: quorum spans both, so no op
+ // can commit while the backup misses it, and every reordering-induced
+ // gap blocks quorum until retransmit refills it.
+ server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings {
+ enabled: false,
+ size: iggy_common::IggyByteSize::from(0u64),
+ bucket_capacity: 1,
+ });
+ let replica_count: u8 = 2;
+ let mut sim = Simulator::new(
+ replica_count as usize,
+ std::iter::once(CLIENT_ID),
+ packet::PacketSimulatorOptions {
+ node_count: replica_count,
+ client_count: 1,
+ seed: 0x5EED_0243,
+ ..packet::PacketSimulatorOptions::default()
+ },
+ );
+ let client = SimClient::new(CLIENT_ID);
+ sim.register_client_with_primary(&client);
+ REPAIR_REQUESTS.store(0, Ordering::Relaxed);
+
+ for (from, to) in [(0u8, 1u8), (1, 0)] {
+ *sim.network
+ .link_drop_packet_fn(ProcessId::Replica(from), ProcessId::Replica(to)) =
+ Some(count_repair_requests);
+ }
+
+ // Sustained, not bursty, and the run asserts the load went somewhere,
+ // so a green result cannot come from a workload that never loaded the
+ // group.
+ let mut lag_run = 0u32;
+ let mut longest_lag_run = 0u32;
+ let sample = |sim: &Simulator, lag_run: &mut u32, longest: &mut u32| {
+ let (_, _, commit_min, commit_max) = metadata_state(sim, 1);
+ if commit_min < commit_max {
+ *lag_run += 1;
+ *longest = (*longest).max(*lag_run);
+ } else {
+ *lag_run = 0;
+ }
+ };
+ for tick in 0..LOAD_TICKS {
+ if tick % TICKS_PER_SEND == 0 {
+ let msg = client.create_stream(&format!("healthy-{tick}"));
+ sim.submit_request(client.client_id(), 0, msg.into_generic());
+ }
+ sim.step();
+ sample(&sim, &mut lag_run, &mut longest_lag_run);
+ }
+ for _ in 0..QUIET_STEPS {
+ sim.step();
+ sample(&sim, &mut lag_run, &mut longest_lag_run);
+ }
+
+ let committed = metadata_state(&sim, 1).2;
+ let sends = LOAD_TICKS / TICKS_PER_SEND;
+ assert!(
+ committed >= COMMITTED_MIN,
+ "the backup committed only {committed} ops across {sends} sends, so the \
+ driver was never ticked over a loaded group"
+ );
+ // Recorded, not load-bearing: the tail walk in `on_replicate` keeps a
+ // healthy backup caught up at quiescence, so a naive lag detector has
+ // nothing to misread here; `gap_detector_tests` pins the predicate.
+ assert_eq!(
+ longest_lag_run, 0,
+ "healthy two-replica metadata traffic left the backup lagging for \
+ {longest_lag_run} consecutive ticks; if this ever becomes non-zero \
+ the predicate's journal-hole half is load-bearing HERE too and this \
+ test should assert against it rather than record it"
+ );
+ for replica in 0..replica_count {
+ let (status, view, commit_min, commit_max) = metadata_state(&sim, replica);
+ assert_eq!(
+ (status, view),
+ (Status::Normal, 0),
+ "replica {replica} left view 0 / Normal, so a view change could \
+ account for repair traffic"
+ );
+ if commit_min < commit_max {
+ assert!(
+ journal_holds(&sim, replica, commit_min + 1),
+ "replica {replica} lags at {commit_min} of {commit_max} with op \
+ {} missing, so a no-loss run produced a real hole",
+ commit_min + 1
+ );
+ }
+ }
+ assert_eq!(
+ REPAIR_REQUESTS.load(Ordering::Relaxed),
+ 0,
+ "the tick driver requested repair on a healthy group; its gap predicate \
+ is reading ordinary commit lag as a journal hole"
+ );
+ }
+}