You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/26 22:58:24 UTC

(iggy) branch master updated: test(partitions): expose bugs when a slow replica rejoins the cluster (#4653)

This is an automated email from the ASF dual-hosted git repository.

hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git


The following commit(s) were added to refs/heads/master by this push:
     new 8c9b9d2e5 test(partitions): expose bugs when a slow replica rejoins the cluster (#4653)
8c9b9d2e5 is described below

commit 8c9b9d2e59d1358679839393bdd81085f424bbbf
Author: Maciej Modzelewski <ma...@modzelewski.dev>
AuthorDate: Mon Oct 26 23:58:19 2026 +0100

    test(partitions): expose bugs when a slow replica rejoins the cluster (#4653)
---
 .../tests/cluster/partition_state_transfer.rs      | 201 +++++++++++++++++-
 core/shard/src/lib.rs                              | 234 +++++++++++++++++++++
 core/simulator/src/lib.rs                          |   3 +
 core/simulator/src/network.rs                      |   6 +
 core/simulator/src/packet.rs                       |  76 ++++++-
 core/simulator/src/slow_replica_tests.rs           | 148 +++++++++++++
 6 files changed, 663 insertions(+), 5 deletions(-)

diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs
index 36716ffa6..4decebf09 100644
--- a/core/integration/tests/cluster/partition_state_transfer.rs
+++ b/core/integration/tests/cluster/partition_state_transfer.rs
@@ -27,13 +27,18 @@
 //! `RangeEvicted`, the repaired window cannot connect to recovered state,
 //! and `complete_repair` returns the `FloorRefused` conversion trigger.
 
+use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 
 use iggy::prelude::*;
-use integration::harness::TestHarness;
+use integration::harness::{TestHarness, TestServerConfig, resolve_config_paths};
 use integration::iggy_harness;
+use serial_test::parallel;
+use tokio::task::JoinHandle;
 use tokio::time::sleep;
 
 const STREAM_NAME: &str = "partition-transfer-stream";
@@ -68,6 +73,9 @@ const KILL_GATE_POLL: Duration = Duration::from_millis(10);
 
 const CONVERSION_MARKER: &str = "partition repair floor unreachable; converting to state transfer";
 const INSTALL_MARKER: &str = "partition state transfer installed";
+const INSTALL_FAILED_MARKER: &str = "partition state transfer install failed";
+/// The `StaleTransfer` reason, logged inside an [`INSTALL_FAILED_MARKER`] line.
+const STALE_INSTALL_MARKER: &str = "is below the local commit frontier";
 /// Logged once per transfer, at the last byte of the last artifact. The
 /// descriptor-time "serving" line only proves a request ARRIVED; this proves the
 /// pull ran.
@@ -84,6 +92,17 @@ const ABANDON_MARKER: &str =
 const REARM_ON_VIEW_MARKER: &str =
     "adopted a live view while awaiting transfer; requesting partition state transfer";
 
+/// Sustained-write spec: the node cut off and then healed.
+const LAGGING_NODE: usize = 2;
+/// Sends committed while node 2 is cut off: more than the evicted ring (capacity
+/// 64) holds, so its repair after the heal cannot reach its own commit point.
+const ISOLATED_SENDS: u32 = 100;
+/// Pace of the writer that keeps running after the heal. One message a second
+/// already moves the commit point past every offer before its pull lands.
+const WRITE_INTERVAL: Duration = Duration::from_secs(1);
+/// How long node 2 must go without another conversion once it installed.
+const CAUGHT_UP_WINDOW: Duration = Duration::from_secs(10);
+
 /// Transfer end-to-end: adoption, repair round-trip, conversion, chunk pull,
 /// install, tail repair. CI runners are slow; bound without hanging the suite.
 const TRANSFER_BUDGET: Duration = Duration::from_secs(60);
@@ -353,6 +372,103 @@ async fn given_transfer_peer_dies_when_stalled_should_leave_dead_peer_and_recove
     }
 }
 
+/// A backup that falls past the evicted ring must rejoin through one state
+/// transfer while clients keep writing, and then stay caught up.
+///
+/// Node 2 is cut off while the partition commits more ops than the ring holds,
+/// so its repair after the heal hits `RangeEvicted` and converts to a transfer.
+/// One message per second keeps arriving for the whole run. That is enough to
+/// keep the cluster's commit point moving during every pull. No bulky seed is
+/// needed: the pull itself is a few KiB.
+#[tokio::test(flavor = "multi_thread")]
+#[parallel]
+#[ignore = "known bug: under sustained writes the backup refuses every transfer as stale (transfer frontier below the local commit frontier); run with --run-ignored only"]
+async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_rejoin_through_one_transfer()
+ {
+    let mut harness = relayed_cluster();
+    harness.start().await.expect("start the cluster");
+    let client = harness
+        .root_client_for_node(0)
+        .await
+        .expect("connect a root client to the node");
+    seed_topic(&client, None).await;
+
+    harness
+        .isolate_node(LAGGING_NODE)
+        .expect("cut node 2 off from its peers");
+    produce(&client, ISOLATED_SENDS).await;
+    let writer = SteadyWriter::start(
+        harness
+            .root_client_for_node(0)
+            .await
+            .expect("connect the steady writer"),
+    );
+    harness
+        .heal_node(LAGGING_NODE)
+        .expect("restore node 2's peer links");
+    await_marker(&harness, LAGGING_NODE, CONVERSION_MARKER).await;
+
+    // Today every install is refused with "transfer frontier X is below the
+    // local commit frontier X+1". The first request only pins the offer at the
+    // primary's commit point (the build answers "flush is pending"), and the
+    // pull runs on the next request, one `repair_retry_interval` later. Node 2
+    // journals the op committed in between, and the install refuses an offer
+    // below an op it holds.
+    let deadline = Instant::now() + TRANSFER_BUDGET;
+    while harness
+        .node(LAGGING_NODE)
+        .stdout_occurrences(INSTALL_MARKER)
+        == 0
+    {
+        assert!(
+            Instant::now() < deadline,
+            "node 2 fell past the evicted ring and never installed a state transfer within \
+             {TRANSFER_BUDGET:?} while one message per second was written: the peers fully \
+             served {} pull(s) and node 2 failed {} install(s), {} of them as stale, so a \
+             backup cannot rejoin under sustained writes",
+            fully_served_count(&harness),
+            harness
+                .node(LAGGING_NODE)
+                .stdout_occurrences(INSTALL_FAILED_MARKER),
+            harness
+                .node(LAGGING_NODE)
+                .stdout_occurrences(STALE_INSTALL_MARKER),
+        );
+        sleep(MARKER_POLL).await;
+    }
+
+    // Installed: the tail above the offer is now ordinary journal repair. A
+    // backup that keeps up never converts to a transfer again.
+    let conversions = harness
+        .node(LAGGING_NODE)
+        .stdout_occurrences(CONVERSION_MARKER);
+    let acked_at_install = writer.acked();
+    let window_end = Instant::now() + CAUGHT_UP_WINDOW;
+    while Instant::now() < window_end {
+        let converted = harness
+            .node(LAGGING_NODE)
+            .stdout_occurrences(CONVERSION_MARKER)
+            - conversions;
+        assert_eq!(
+            converted, 0,
+            "node 2 installed a state transfer, then fell past the evicted ring again and \
+             converted to another transfer while writes continued"
+        );
+        sleep(MARKER_POLL).await;
+    }
+    // The markers only say the machinery ran. Every send committed while node
+    // 2 was cut off must be in its own segment bytes, in order.
+    let expected: Vec<String> = (0..ISOLATED_SENDS)
+        .map(|sequence| format!("message-{sequence}"))
+        .collect();
+    await_installed_payloads(&harness, LAGGING_NODE, &expected).await;
+    let acked = writer.stop().await;
+    assert!(
+        acked > acked_at_install,
+        "the steady writer must keep committing through the caught-up window"
+    );
+}
+
 async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) {
     // Node 2 holds a durable prefix, then misses enough traffic that the
     // survivors' ring moves past its durable end: its repaired window cannot
@@ -395,6 +511,89 @@ async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) {
     await_installed_payloads(harness, 2, &expected).await;
 }
 
+/// The specs above go through `iggy_harness`, which cannot relay the replica
+/// links, so the one that cuts a node off builds its cluster here with the same
+/// server settings.
+fn relayed_cluster() -> TestHarness {
+    let overrides = HashMap::from([
+        ("sharding.cpu_allocation".to_owned(), "0..1".to_owned()),
+        (
+            "partition.evicted_ring_capacity".to_owned(),
+            "64".to_owned(),
+        ),
+    ]);
+    let extra_envs = resolve_config_paths(&overrides).expect("valid server config paths");
+    TestHarness::builder()
+        .server(TestServerConfig::builder().extra_envs(extra_envs).build())
+        .cluster_nodes(3)
+        .peer_relays()
+        .build()
+        .expect("build the harness")
+}
+
+/// Sends one small message per [`WRITE_INTERVAL`] from its own task until
+/// stopped. A failed send is not retried: the next tick sends a new message,
+/// and only acknowledged sends are counted.
+struct SteadyWriter {
+    stop: Arc<AtomicBool>,
+    acked: Arc<AtomicU64>,
+    task: JoinHandle<()>,
+}
+
+impl SteadyWriter {
+    fn start(client: IggyClient) -> Self {
+        let stop = Arc::new(AtomicBool::new(false));
+        let acked = Arc::new(AtomicU64::new(0));
+        let task = tokio::spawn({
+            let stop = Arc::clone(&stop);
+            let acked = Arc::clone(&acked);
+            async move {
+                let stream = Identifier::named(STREAM_NAME).expect("stream identifier");
+                let topic = Identifier::named(TOPIC_NAME).expect("topic identifier");
+                let mut sequence = 0_u64;
+                while !stop.load(Ordering::Relaxed) {
+                    let mut messages = vec![
+                        IggyMessage::from_str(&format!("steady-{sequence}")).expect("message"),
+                    ];
+                    sequence += 1;
+                    if client
+                        .send_messages(
+                            &stream,
+                            &topic,
+                            &Partitioning::partition_id(PARTITION_ID),
+                            &mut messages,
+                        )
+                        .await
+                        .is_ok()
+                    {
+                        acked.fetch_add(1, Ordering::Relaxed);
+                    }
+                    sleep(WRITE_INTERVAL).await;
+                }
+            }
+        });
+        Self { stop, acked, task }
+    }
+
+    fn acked(&self) -> u64 {
+        self.acked.load(Ordering::Relaxed)
+    }
+
+    async fn stop(self) -> u64 {
+        self.stop.store(true, Ordering::Relaxed);
+        self.task.await.expect("the steady writer task ends");
+        self.acked.load(Ordering::Relaxed)
+    }
+}
+
+/// Pulls the survivors served to their last byte, whoever served them.
+fn fully_served_count(harness: &TestHarness) -> usize {
+    (0..harness.cluster_size())
+        .filter(|&node| node != LAGGING_NODE)
+        .map(|node| harness.node(node).stdout_occurrences(FULLY_SERVED_MARKER))
+        .sum()
+}
+
 async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option<IggyClient> {
     for &node in nodes {
         if let Ok(builder) = harness.node(node).tcp_client()
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 1c35bb54f..e6b166200 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -14498,6 +14498,240 @@ mod dvc_suffix_window_tests {
     }
 }
 
+#[cfg(test)]
+mod installed_transfer_dvc_tests {
+    //! A lagging replica adopts a `StartView` and later installs a state
+    //! transfer below that view's head. The install rewinds the sequencer to
+    //! the transfer's op and keeps the adopted view. Each `DoViewChange` the
+    //! replica sends from then on carries the transfer op in its header but
+    //! starts its suffix at the adopted head, and its peers drop every one as
+    //! unreadable. With a third replica down, or back with an untrusted log,
+    //! no quorum of readable votes forms and the view change never ends.
+
+    use super::*;
+    use consensus::{LocalPipeline, dvc_suffix_decode};
+    use iggy_common::{IggyByteSize, PartitionStats};
+    use journal::prepare_journal::PrepareJournal;
+    use message_bus::IggyMessageBus;
+    use server_common::iobuf::Owned;
+    use std::sync::Arc;
+
+    const CLUSTER: u128 = 1;
+    const REPLICA_COUNT: u8 = 3;
+    const GROUP: u64 = 42;
+    /// Adopts the view, then installs the transfer.
+    const LAGGING: u8 = 2;
+    /// Led by replica 0.
+    const ADOPTED_VIEW: u32 = 3;
+    /// Led by replica 1, which receives the lagging replica's vote.
+    const NEXT_VIEW: u32 = 4;
+    const HEAD: u64 = 300;
+    /// Two below the head, so the adopted view still carries uncommitted ops.
+    const COMMIT: u64 = HEAD - 2;
+
+    #[compio::test]
+    #[ignore = "known bug: a transfer installed below the adopted view head makes the replica's DoViewChange unreadable; run with --ignored"]
+    async fn given_a_transfer_installed_below_the_adopted_head_when_a_view_change_follows_should_send_a_readable_do_view_change()
+     {
+        // Inside the 128-entry window, so only the head disagrees.
+        let transfer = HEAD - 100;
+        // Boxed: the setup future carries a whole partition.
+        let (vote, message) = Box::pin(vote_after_install(transfer)).await;
+        let VsrAction::SendDoViewChange { suffix, .. } = &vote else {
+            unreachable!("matched in vote_after_install");
+        };
+        let header = message.header();
+        // What `decode_peer_suffix` runs on the receiving replica.
+        if let Err(error) = dvc_suffix_decode(
+            &message.as_slice()[size_of::<DoViewChangeHeader>()..],
+            header.op,
+            header.nack_bitset,
+            header.present_bitset,
+        ) {
+            panic!(
+                "replica {LAGGING} adopted a view with head {HEAD} and then installed a state \
+                 transfer at op {transfer}. Its DoViewChange carries op {} in the header but starts \
+                 its suffix at op {}, so the receiver drops it as unreadable ({error}) and the \
+                 view change cannot complete without a vote from the third replica",
+                header.op,
+                suffix.headers().first().map_or(0, |entry| entry.op),
+            );
+        }
+    }
+
+    #[compio::test]
+    #[ignore = "known bug: a transfer installed far below the adopted view head leaves uncommitted ops outside the DoViewChange window; run with --ignored"]
+    async fn given_a_transfer_installed_far_below_the_adopted_head_when_a_view_change_follows_should_describe_every_uncommitted_op()
+     {
+        // Past the 128-entry window.
+        let transfer = HEAD - 200;
+        let (vote, _) = Box::pin(vote_after_install(transfer)).await;
+        let VsrAction::SendDoViewChange { commit, suffix, .. } = &vote else {
+            unreachable!("matched in vote_after_install");
+        };
+        let floor = suffix.headers().last().map_or(0, |entry| entry.op);
+        assert!(
+            floor <= *commit,
+            "replica {LAGGING}'s DoViewChange window starts at op {floor}, above its commit point \
+             {commit}, so ops {}..={} go out described by nobody and the view change cannot decide \
+             them. The install put the sequencer {} ops below the adopted head, past the \
+             {DVC_HEADERS_MAX}-entry window that `build_dvc_suffix` treats as unreachable",
+            commit + 1,
+            floor.saturating_sub(1),
+            HEAD - transfer,
+        );
+    }
+
+    /// The lagging replica's vote for `NEXT_VIEW`, and the frame the shard sends
+    /// for it. Before the vote, the replica adopts the `ADOPTED_VIEW` `StartView`
+    /// and then installs a peer's state transfer at `transfer`.
+    #[allow(clippy::future_not_send)]
+    async fn vote_after_install(transfer: u64) -> (VsrAction, Message<DoViewChangeHeader>) {
+        let bus = IggyMessageBus::new(0);
+        let sent = Rc::new(RefCell::new(Vec::new()));
+        let captured = Rc::clone(&sent);
+        bus.set_replica_forward_fn(Box::new(move |_, _, frame| {
+            captured.borrow_mut().push(frame);
+            Ok(())
+        }));
+        for peer in (0..REPLICA_COUNT).filter(|&peer| peer != LAGGING) {
+            assert!(bus.owner_table().try_claim(peer, 1));
+        }
+        let consensus = VsrConsensus::new(
+            CLUSTER,
+            LAGGING,
+            REPLICA_COUNT,
+            GROUP,
+            bus,
+            LocalPipeline::new(),
+        );
+        consensus.init();
+        let mut partition: IggyPartition<IggyMessageBus> = IggyPartition::with_in_memory_storage(
+            Arc::new(PartitionStats::default()),
+            consensus,
+            IggyByteSize::from(1024 * 1024),
+        );
+
+        // `on_start_view`: adopt, then reconcile the journal against the view.
+        let announce = start_view();
+        let actions = partition
+            .consensus()
+            .try_handle_start_view(
+                PlaneKind::Partitions,
+                announce.header(),
+                &announce.as_slice()[size_of::<StartViewHeader>()..],
+                |op| partition.applied_header(op),
+            )
+            .expect("the StartView is well formed");
+        assert!(!actions.is_empty(), "the lagging replica adopts the view");
+        let pending = partition.consensus().pending_view_log();
+        reconcile_partition_view_divergence(0, &mut partition, pending.as_ref()).await;
+
+        // Repair finds the gap evicted at every peer, so a transfer follows. The
+        // install cannot run here: its offsets artifact is a type private to
+        // `partitions`. So these are its journal wipe and then the steps of
+        // `publish_installed_consensus`, in its order, with a checkpoint prepare
+        // built here in place of the origin's. They rewind the sequencer to
+        // `transfer` and keep the adopted view.
+        let checkpoint = Message::<PrepareHeader>::new(size_of::<PrepareHeader>())
+            .transmute_header(|_, header: &mut PrepareHeader| {
+                *header = prepare_header(transfer, 0);
+            });
+        partition.log.journal().inner.clear_all();
+        partition.log.journal().inner.restore_checkpoint_prepare(
+            transfer,
+            Owned::<MESSAGE_ALIGN>::copy_from_slice(checkpoint.as_slice()).into(),
+        );
+        let consensus = partition.consensus();
+        consensus.set_commit_floor(transfer);
+        consensus.sequencer().set_sequence(transfer);
+        consensus.set_last_prepare_checksum(checkpoint.header().checksum);
+        consensus.clear_pipeline();
+        consensus.advance_commit_max(transfer);
+
+        // `on_start_view_change`: snapshot the suffix, then vote.
+        refresh_partition_dvc_suffix(&partition);
+        let consensus = partition.consensus();
+        let actions =
+            consensus.handle_start_view_change(PlaneKind::Partitions, start_view_change().header());
+        let vote = actions
+            .iter()
+            .find(|action| matches!(action, VsrAction::SendDoViewChange { .. }))
+            .cloned()
+            .expect("the lagging replica votes");
+        dispatch_partition_wire_actions::<_, _, PrepareJournal, _>(consensus, &partition, actions)
+            .await;
+        let message = sent
+            .borrow()
+            .iter()
+            .find_map(|frame| {
+                Message::<DoViewChangeHeader>::try_from(Owned::copy_from_slice(frame.as_slice()))
+                    .ok()
+            })
+            .expect("the vote goes out on the wire");
+        (vote, message)
+    }
+
+    /// The `ADOPTED_VIEW` `StartView`: its head and commit point, with the
+    /// hash-chained headers of `COMMIT..=HEAD`, high-to-low.
+    fn start_view() -> Message<StartViewHeader> {
+        let mut headers = Vec::new();
+        let mut parent = 0;
+        for op in COMMIT..=HEAD {
+            let header = prepare_header(op, parent);
+            parent = header.checksum;
+            headers.push(header);
+        }
+        headers.reverse();
+        let header_size = size_of::<StartViewHeader>();
+        let total = header_size + size_of_val(headers.as_slice());
+        let mut message = Message::<StartViewHeader>::new(total);
+        encode_prepare_headers(&headers, &mut message.as_mut_slice()[header_size..total]);
+        message.transmute_header(|_, header: &mut StartViewHeader| {
+            header.command = Command::StartView;
+            header.cluster = CLUSTER;
+            header.view = ADOPTED_VIEW;
+            header.op = HEAD;
+            header.commit = COMMIT;
+            header.replica = 0;
+            header.group = GROUP;
+            header.size = u32::try_from(total).unwrap();
+        })
+    }
+
+    fn start_view_change() -> Message<StartViewChangeHeader> {
+        Message::<StartViewChangeHeader>::new(size_of::<StartViewChangeHeader>()).transmute_header(
+            |_, header: &mut StartViewChangeHeader| {
+                header.command = Command::StartViewChange;
+                header.cluster = CLUSTER;
+                header.view = NEXT_VIEW;
+                header.replica = 1;
+                header.group = GROUP;
+                header.size = u32::try_from(size_of::<StartViewChangeHeader>()).unwrap();
+            },
+        )
+    }
+
+    /// A body-less prepare header at `op` in `ADOPTED_VIEW`, chained to `parent`.
+    fn prepare_header(op: u64, parent: u128) -> PrepareHeader {
+        let mut header = PrepareHeader {
+            command: Command::Prepare,
+            cluster: CLUSTER,
+            group: GROUP,
+            operation: Operation::StoreConsumerOffset,
+            size: u32::try_from(size_of::<PrepareHeader>()).unwrap(),
+            op,
+            commit: op - 1,
+            timestamp: op,
+            parent,
+            view: ADOPTED_VIEW,
+            ..PrepareHeader::default()
+        };
+        header.checksum = header.identity_checksum();
+        header
+    }
+}
+
 #[cfg(test)]
 mod adopted_view_retention_tests {
     //! The view 12 stall of workload-fuzz seed 890 at 45a4ea297. Replica 0
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 199221480..4d5329eec 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -2221,6 +2221,9 @@ fn stamp_metadata_watermark(
     }
 }
 
+#[cfg(test)]
+mod slow_replica_tests;
+
 #[cfg(test)]
 mod stale_prepare_commit_tests;
 
diff --git a/core/simulator/src/network.rs b/core/simulator/src/network.rs
index 43984693c..890d62a33 100644
--- a/core/simulator/src/network.rs
+++ b/core/simulator/src/network.rs
@@ -153,6 +153,12 @@ impl Network {
         self.simulator.link_drop_packet_fn(from, to)
     }
 
+    /// Returns a mutable reference to the link's extra one-way delay, in ticks.
+    /// See [`PacketSimulator::link_extra_delay`].
+    pub fn link_extra_delay(&mut self, from: ProcessId, to: ProcessId) -> &mut u64 {
+        self.simulator.link_extra_delay(from, to)
+    }
+
     /// Clog a specific link (bidirectionally).
     ///
     /// Clogged links do not deliver any packets until unclogged.
diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs
index ba648636a..30417e200 100644
--- a/core/simulator/src/packet.rs
+++ b/core/simulator/src/packet.rs
@@ -19,6 +19,7 @@
 //!
 //! This module provides a packet-level network simulator that models:
 //! - Per-path latency with exponential delay distribution
+//! - Per-link extra latency, directed, set by the caller
 //! - Packet loss and replay (duplication) at delivery time
 //! - Automatic network partitioning with configurable lifecycle
 //! - Automatic path clogging with exponential duration
@@ -237,6 +238,9 @@ struct Link {
     /// Optional predicate to drop specific packets. Checked after the link filter
     /// but before the random loss check.
     drop_packet_fn: Option<fn(&Packet) -> bool>,
+    /// Ticks added to the drawn one-way delay of every packet submitted on this
+    /// link, replays included. Zero by default.
+    extra_delay: u64,
 }
 
 #[allow(clippy::missing_fields_in_debug)]
@@ -247,6 +251,7 @@ impl std::fmt::Debug for Link {
             .field("clogged_till", &self.clogged_till)
             .field("filter", &self.filter)
             .field("drop_packet_fn", &self.drop_packet_fn.map(|_| "<fn>"))
+            .field("extra_delay", &self.extra_delay)
             .finish()
     }
 }
@@ -258,6 +263,7 @@ impl Link {
             clogged_till: 0,
             filter: ALLOW_ALL,
             drop_packet_fn: None,
+            extra_delay: 0,
         }
     }
 }
@@ -527,7 +533,9 @@ impl PacketSimulator {
     /// time in `step()`. If the link is at capacity, a random existing packet
     /// is evicted to make room.
     pub fn submit(&mut self, from: ProcessId, to: ProcessId, message: Message<GenericHeader>) {
-        let delay = Self::calculate_delay(&mut self.prng, &self.options);
+        let idx = self.link_index(from, to);
+        let delay = Self::calculate_delay(&mut self.prng, &self.options)
+            .saturating_add(self.links[idx].extra_delay);
         let ready_at = self.current_tick.saturating_add(delay);
 
         let packet = Packet {
@@ -537,8 +545,6 @@ impl PacketSimulator {
             ready_at,
         };
 
-        let idx = self.link_index(from, to);
-
         // If at capacity, evict a random existing packet
         if self.links[idx].packets.len() >= self.options.link_capacity as usize {
             self.links[idx].packets.remove_random(&mut self.prng);
@@ -614,6 +620,21 @@ impl PacketSimulator {
         &mut self.links[idx].drop_packet_fn
     }
 
+    /// Returns a mutable reference to the link's extra one-way delay, in ticks.
+    ///
+    /// Directed: only packets from `from` to `to` wait longer. The extra delay
+    /// is added to the delay drawn from `one_way_delay_min` and
+    /// `one_way_delay_mean` without drawing from the PRNG, so a run with every
+    /// extra delay at zero replays the same trace. Setting it on every link out
+    /// of one replica models a slow egress on that node, as `tc netem delay`
+    /// does. A delayed link holds its packets longer, so `link_capacity` evicts
+    /// sooner on it. [`Self::heal`] leaves it in place, like
+    /// [`Self::link_drop_packet_fn`].
+    pub fn link_extra_delay(&mut self, from: ProcessId, to: ProcessId) -> &mut u64 {
+        let idx = self.link_index(from, to);
+        &mut self.links[idx].extra_delay
+    }
+
     /// Mark a process down. Anything addressed to it is dropped at delivery.
     ///
     /// Link filters are untouched, so a partition or command filter standing at crash
@@ -732,7 +753,8 @@ impl PacketSimulator {
 
                     // Random replay check: clone and re-enqueue with eviction at capacity
                     if prng.random::<f64>() < options.replay_probability {
-                        let delay = Self::calculate_delay(prng, options);
+                        let delay =
+                            Self::calculate_delay(prng, options).saturating_add(link.extra_delay);
                         let replay = Packet {
                             ready_at: (*current_tick).saturating_add(delay),
                             ..packet.clone()
@@ -1147,6 +1169,52 @@ mod tests {
         assert_eq!(delivered.len(), 1);
     }
 
+    /// An extra delay on one link holds back only that direction, by at least
+    /// the extra ticks, and the reverse link keeps the drawn delay.
+    #[test]
+    fn given_extra_link_delay_when_submitting_should_delay_only_that_direction() {
+        const EXTRA: u64 = 30;
+        let options = PacketSimulatorOptions {
+            one_way_delay_min: 1,
+            one_way_delay_mean: 1,
+            node_count: 2,
+            client_count: 0,
+            seed: 42,
+            ..Default::default()
+        };
+
+        let mut sim = PacketSimulator::new(options);
+        *sim.link_extra_delay(ProcessId::Replica(0), ProcessId::Replica(1)) = EXTRA;
+        sim.submit(
+            ProcessId::Replica(0),
+            ProcessId::Replica(1),
+            create_test_message(),
+        );
+        sim.submit(
+            ProcessId::Replica(1),
+            ProcessId::Replica(0),
+            create_test_message(),
+        );
+
+        let mut delivered_at = HashMap::new();
+        for tick in 0..=EXTRA + 20 {
+            for packet in sim.step() {
+                delivered_at.insert(packet.from, tick);
+            }
+            sim.tick();
+        }
+        let slow = delivered_at[&ProcessId::Replica(0)];
+        let fast = delivered_at[&ProcessId::Replica(1)];
+        assert!(
+            slow > EXTRA,
+            "the delayed link delivered at tick {slow}, before its extra {EXTRA} ticks"
+        );
+        assert!(
+            fast <= EXTRA,
+            "the reverse link delivered at tick {fast}, so it was delayed too"
+        );
+    }
+
     /// A partition standing when a process crashes still stands when it restarts.
     ///
     /// `process_enable` used to write `ALLOW_ALL` over every link touching the
diff --git a/core/simulator/src/slow_replica_tests.rs b/core/simulator/src/slow_replica_tests.rs
new file mode 100644
index 000000000..0c22f42e3
--- /dev/null
+++ b/core/simulator/src/slow_replica_tests.rs
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! One slow replica must not keep the two healthy replicas changing views.
+//!
+//! Replica 2 is cut off for a moment, long enough to start a view change of its
+//! own, and every message it sends from then on takes longer than the
+//! view-change status timeout to arrive. Its `StartViewChange` for the next
+//! view therefore lands after it already timed out into the view after that.
+//! Replicas 0 and 1 are Normal and commit together, yet each such
+//! `StartViewChange` takes them out of their working view, and the primary
+//! keeps moving between them. Nothing but latency is injected after the cut
+//! heals: no packet is dropped.
+
+use server_common::sharding::IggyNamespace;
+
+use crate::Simulator;
+use crate::packet::ProcessId;
+use crate::partition_repair_driver_tests::cluster;
+use crate::workload::{
+    self, Workload,
+    invariants::Invariants,
+    options::{ActionWeights, WorkloadOptions},
+};
+
+const SEED: u64 = 0x5052_494D;
+const SLOW: u8 = 2;
+/// Groups the rebalancer spreads across the replicas, so each replica leads
+/// one before the fault.
+const GROUPS: usize = 3;
+/// Ticks for the rebalancer to spread the primaries.
+const SPREAD_TICKS: u64 = 10_000;
+/// Past `NORMAL_HEARTBEAT_TICKS` (500): the cut lasts long enough that the
+/// group replica 2 led fails over and replica 2 starts a view change itself.
+const CUT_TICKS: u64 = 600;
+/// One-way delay on every link out of replica 2, just past
+/// `VIEW_CHANGE_STATUS_TICKS` (500). Below that bound the healthy pair adopts
+/// replica 2's view before it times out again, and the views settle.
+const EGRESS_DELAY_TICKS: u64 = 520;
+/// Measured after the heal: 300 s of virtual time at 10 ms per tick.
+const MEASURE_TICKS: u64 = 30_000;
+/// The heal itself may cost a few view changes while replica 2 rejoins: 450
+/// ticks of delay costs up to 3 here, 520 ticks costs about 60.
+const VIEW_CHANGES_MAX: u32 = 5;
+
+#[test]
+#[ignore = "known bug: a replica slower than the view-change status timeout keeps the healthy pair leaving its working view; run with --ignored"]
+fn given_a_replica_slower_than_the_view_change_timeout_when_it_rejoins_should_not_keep_the_healthy_pair_changing_views()
+ {
+    let (mut sim, client) = cluster(SEED);
+    let namespaces: Vec<_> = (0..GROUPS)
+        .map(|partition| IggyNamespace::new(1, 1, partition))
+        .collect();
+    for &namespace in &namespaces {
+        sim.init_partition(namespace);
+    }
+    sim.register_client_with_primary(&client);
+    let mut options = WorkloadOptions::new(SEED, sim.replica_count, namespaces.clone());
+    options.weights = ActionWeights::partition_only();
+    let mut workload = Workload::new(options);
+    let clients = [client];
+    let mut invariants = Invariants::new();
+    let mut tick = |sim: &mut Simulator| {
+        workload::run(sim, &mut workload, &clients, 1, u64::MAX, &mut invariants)
+    };
+
+    for _ in 0..SPREAD_TICKS {
+        tick(&mut sim);
+    }
+    set_peer_links(&mut sim, false);
+    for _ in 0..CUT_TICKS {
+        tick(&mut sim);
+    }
+    set_peer_links(&mut sim, true);
+    for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) {
+        *sim.network
+            .link_extra_delay(ProcessId::Replica(SLOW), ProcessId::Replica(peer)) =
+            EGRESS_DELAY_TICKS;
+    }
+
+    let first_views: Vec<u32> = namespaces
+        .iter()
+        .map(|&namespace| healthy_view(&sim, namespace))
+        .collect();
+    let mut views = first_views.clone();
+    let mut changes = vec![0_u32; GROUPS];
+    let mut replies = 0;
+    for _ in 0..MEASURE_TICKS {
+        replies += tick(&mut sim);
+        for (index, &namespace) in namespaces.iter().enumerate() {
+            let view = healthy_view(&sim, namespace);
+            if view != views[index] {
+                changes[index] += 1;
+                views[index] = view;
+            }
+        }
+    }
+    assert!(
+        replies > 0,
+        "the workload must keep committing after the heal"
+    );
+    for (index, &namespace) in namespaces.iter().enumerate() {
+        assert!(
+            changes[index] <= VIEW_CHANGES_MAX,
+            "replicas 0 and 1 changed view {} times for {namespace:?} in {MEASURE_TICKS} ticks \
+             (view {} -> {}), while only replica 2 was slow: one replica whose messages \
+             arrive later than the view-change status timeout keeps the healthy pair leaving \
+             its working view (changes per group {changes:?})",
+            changes[index],
+            first_views[index],
+            views[index],
+        );
+    }
+}
+
+/// Cuts or restores every link between the slow replica and its peers.
+fn set_peer_links(sim: &mut Simulator, enabled: bool) {
+    for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) {
+        sim.network
+            .set_link_filter(ProcessId::Replica(SLOW), ProcessId::Replica(peer), enabled);
+        sim.network
+            .set_link_filter(ProcessId::Replica(peer), ProcessId::Replica(SLOW), enabled);
+    }
+}
+
+/// The highest view replicas 0 and 1 hold for `namespace`.
+fn healthy_view(sim: &Simulator, namespace: IggyNamespace) -> u32 {
+    (0..sim.replica_count)
+        .filter(|&replica| replica != SLOW)
+        .filter_map(|replica| sim.partition_consensus_state(usize::from(replica), namespace))
+        .map(|state| state.view)
+        .max()
+        .unwrap_or(0)
+}