You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/10/23 18:45:59 UTC
(iggy) branch master updated: fix(partitions): don't commit a stale prepare after adopting a newer view (#4433)
This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 5d51cb6c8 fix(partitions): don't commit a stale prepare after adopting a newer view (#4433)
5d51cb6c8 is described below
commit 5d51cb6c834fb45a965536c915d3297c90ffdc42
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Fri Oct 23 20:45:54 2026 +0200
fix(partitions): don't commit a stale prepare after adopting a newer view (#4433)
Closes #4284
Co-authored-by: Mark Aron Szulyovszky <ma...@gmail.com>
---
core/binary_protocol/src/consensus/header.rs | 8 +
core/configs/src/server_config/cluster.rs | 2 +-
core/consensus/src/dvc_merge.rs | 25 +-
core/consensus/src/impls.rs | 9 +
core/integration/tests/cluster/crash_durability.rs | 96 ++
core/integration/tests/cluster/mod.rs | 1 +
.../tests/cluster/rolling_sigkill_repro.rs | 320 +++++
core/partitions/src/iggy_partition.rs | 1306 +++++++++++++++++++-
core/partitions/src/journal.rs | 59 +
core/partitions/src/lib.rs | 2 +
core/partitions/src/state_transfer.rs | 1 +
core/partitions/src/types.rs | 12 +-
core/partitions/src/view_fence.rs | 166 +++
core/server/config.toml | 5 +-
core/server/src/partition_reconciler.rs | 1 +
core/shard/src/lib.rs | 505 +++++++-
core/shard/src/partition_io.rs | 178 ++-
core/simulator/src/lib.rs | 5 +-
core/simulator/src/stale_prepare_commit_tests.rs | 505 ++++++++
19 files changed, 3106 insertions(+), 100 deletions(-)
diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs
index e46150f17..222c2b33e 100644
--- a/core/binary_protocol/src/consensus/header.rs
+++ b/core/binary_protocol/src/consensus/header.rs
@@ -1260,6 +1260,14 @@ impl PrepareHeader {
covered.view = 0;
u128::from(twox_hash::XxHash3_64::oneshot(bytemuck::bytes_of(&covered)))
}
+
+ /// Whether a parent chain can be proven down from this header. An unsealed
+ /// header names no identity, and a zero `parent` would link anything below it;
+ /// op 1 is the one prepare whose real parent is zero.
+ #[must_use]
+ pub const fn anchors_chain(&self) -> bool {
+ self.checksum != CHECKSUM_UNSEALED && (self.parent != CHECKSUM_UNSEALED || self.op == 1)
+ }
}
// RepairPrepareHeader - repair peer -> recovering replica (journal repair)
diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs
index 280d1eb74..0848cb9e7 100644
--- a/core/configs/src/server_config/cluster.rs
+++ b/core/configs/src/server_config/cluster.rs
@@ -296,7 +296,7 @@ pub struct ClusterConfig {
#[serde_as(as = "DisplayFromStr")]
#[config_env(leaf)]
pub repair_gap_debounce_interval: IggyDuration,
- /// Prepares a peer serves per repair round before the requester walks to
+ /// Prepares a peer serves per repair round before the requester asks for
/// the next chunk. Each frame rides the per-peer message-bus queue, so this
/// must stay below `message_bus.peer_queue_capacity` or a full round
/// overruns the queue and silently drops frames (enforced at the top
diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs
index da76f0549..33d5ac602 100644
--- a/core/consensus/src/dvc_merge.rs
+++ b/core/consensus/src/dvc_merge.rs
@@ -28,9 +28,10 @@
#[cfg(test)]
use crate::view_change_quorum::DvcSuffix;
use crate::view_change_quorum::{
- DvcQuorumArray, StoredDvc, dvc_count, dvc_iter, dvc_quorum_array_empty, dvc_record,
+ DvcQuorumArray, DvcSuffixError, StoredDvc, dvc_count, dvc_iter, dvc_quorum_array_empty,
+ dvc_record, dvc_suffix_decode,
};
-use iggy_binary_protocol::{CHECKSUM_UNSEALED, PrepareHeader};
+use iggy_binary_protocol::{CHECKSUM_UNSEALED, PrepareHeader, StartViewHeader};
/// Sizes the merge needs from the replica.
#[derive(Debug, Clone, Copy)]
@@ -98,6 +99,26 @@ pub struct MergedLog {
pub committed_elsewhere: Vec<PrepareHeader>,
}
+impl MergedLog {
+ /// The log a `StartView` announces: its suffix slots verbatim, blanks included,
+ /// and no headers when it carries numbers only.
+ ///
+ /// # Errors
+ /// Returns the decode error when the suffix body is malformed.
+ pub fn from_start_view(
+ header: &StartViewHeader,
+ suffix_body: &[u8],
+ ) -> Result<Self, DvcSuffixError> {
+ let suffix = dvc_suffix_decode(suffix_body, header.op, 0, 0)?;
+ Ok(Self {
+ op_head: header.op,
+ commit_max: header.commit,
+ headers: suffix.headers().to_vec(),
+ committed_elsewhere: Vec::new(),
+ })
+ }
+}
+
/// Highest op the quorum proves committed.
///
/// Three independent lower bounds, because a single sender's view of the commit
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index cedbd3ba5..04325373e 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -4843,6 +4843,15 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
.flatten()
}
+ /// Whether the accepted `StartView` names `header`'s prepare identity.
+ #[must_use]
+ pub fn adopted_view_vouches_for(&self, header: &PrepareHeader) -> bool {
+ self.adopted_view
+ .borrow()
+ .as_ref()
+ .is_some_and(|adopted| adopted.vouches_for(header))
+ }
+
/// The accepted `StartView`'s view and advertised head, including after repair.
#[must_use]
pub fn adopted_view_head(&self) -> Option<(u32, u64)> {
diff --git a/core/integration/tests/cluster/crash_durability.rs b/core/integration/tests/cluster/crash_durability.rs
index f8ede4894..1ed2a2b13 100644
--- a/core/integration/tests/cluster/crash_durability.rs
+++ b/core/integration/tests/cluster/crash_durability.rs
@@ -251,6 +251,34 @@ async fn wait_until_cluster_serves(
}
}
+/// Poll until `node`'s partition superblock records the highest view any other
+/// node persisted as its log view, so `node` adopted it instead of still probing.
+async fn wait_until_partition_rejoined(harness: &TestHarness, node: usize, budget: Duration) {
+ let deadline = tokio::time::Instant::now() + budget;
+ loop {
+ let view = (0..harness.cluster_size())
+ .filter(|other| *other != node)
+ .filter_map(|other| {
+ disk::read_partition_superblock_state(&harness.node(other).data_path())
+ })
+ .map(|state| state.view)
+ .max();
+ let own = disk::read_partition_superblock_state(&harness.node(node).data_path())
+ .map(|state| (state.view, state.log_view));
+ if let (Some(view), Some((_, log_view))) = (view, own)
+ && log_view >= view
+ {
+ return;
+ }
+ assert!(
+ tokio::time::Instant::now() < deadline,
+ "node {node} did not rejoin the partition's view within {budget:?}: \
+ others at view {view:?}, node {node} at (view, log_view) {own:?}"
+ );
+ sleep(POLL_INTERVAL).await;
+ }
+}
+
/// Poll until every node's segment files hold all of `payloads`, so a
/// follow-up SIGKILL provably erases nothing that was flushed. Replaces a
/// settle sleep: the ack proves quorum COMMIT, but each replica flushes on
@@ -644,6 +672,74 @@ async fn given_persisted_topic_when_backup_misses_writes_should_repair_before_du
.unwrap();
}
+/// Every replica is SIGKILLed and restarted in turn while writes continue, each
+/// missing more ops than one repair chunk. A restarted replica holds back entries
+/// above its recovered commit point until the view's log proves them, so repair has to walk that
+/// window chunk by chunk before the replica can ack again, and the next kill
+/// leaves it as half of the quorum. The next kill waits for it to rejoin the
+/// partition's view: until then it votes as a log that may lack acked ops, and
+/// with one survivor left no trusted quorum can elect.
+#[iggy_harness(cluster_nodes = 3, server(cluster.repair_chunk_max = 4))]
+async fn given_persisted_topic_when_every_replica_is_killed_in_turn_should_preserve_acked_messages(
+ harness: &mut TestHarness,
+) {
+ let client = harness.tcp_root_client().await.unwrap();
+ client
+ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
+ .await
+ .unwrap();
+ client
+ .create_topic(
+ &Identifier::named(STREAM_NAME).unwrap(),
+ TOPIC_NAME,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ durability: Durability::Persisted,
+ ..TopicCreateOptions::default()
+ },
+ )
+ .await
+ .unwrap();
+ let mut acked = produce_acked(&client, "warmup", 16).await;
+ let nodes: Vec<usize> = (0..harness.cluster_size()).collect();
+ for node in nodes.clone() {
+ harness.kill_node(node).unwrap();
+ let survivors: Vec<usize> = nodes.iter().copied().filter(|n| *n != node).collect();
+ let client = wait_until_cluster_serves(harness, &survivors, CONVERGE_TIMEOUT).await;
+ acked.extend(
+ tokio::time::timeout(
+ CONVERGE_TIMEOUT,
+ produce_acked(&client, &format!("node-{node}-down"), 24),
+ )
+ .await
+ .unwrap_or_else(|_| panic!("writes stalled while node {node} was down")),
+ );
+ harness.restart_node(node).unwrap();
+ let client = wait_until_cluster_serves(harness, &nodes, CONVERGE_TIMEOUT).await;
+ acked.extend(
+ tokio::time::timeout(
+ CONVERGE_TIMEOUT,
+ produce_acked(&client, &format!("node-{node}-back"), 8),
+ )
+ .await
+ .unwrap_or_else(|_| panic!("writes stalled after node {node} restarted")),
+ );
+ wait_until_partition_rejoined(harness, node, CONVERGE_TIMEOUT).await;
+ }
+ let client = wait_until_cluster_serves(harness, &nodes, CONVERGE_TIMEOUT).await;
+ wait_for_acked_readable(&client, &acked, CONVERGE_TIMEOUT)
+ .await
+ .unwrap_or_else(|state| {
+ panic!("every acked offset must poll back after the rolling restart: {state}")
+ });
+ let data_paths: Vec<PathBuf> = harness
+ .all_servers()
+ .iter()
+ .map(|server| server.data_path())
+ .collect();
+ disk::wait_for_log_convergence(&data_paths).await;
+}
+
#[iggy_harness(cluster_nodes = 3, server(partition.wal_bytes_max = "134225920 B"))]
async fn given_all_replicas_checkpointed_when_restarted_should_elect_and_extend_the_log(
harness: &mut TestHarness,
diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs
index 546e1d3de..180afda14 100644
--- a/core/integration/tests/cluster/mod.rs
+++ b/core/integration/tests/cluster/mod.rs
@@ -33,6 +33,7 @@ mod partition_primary_routing;
mod partition_state_transfer;
mod register_forwarding;
mod replica_read_batching;
+mod rolling_sigkill_repro;
mod root_credentials;
mod staggered_bootstrap;
mod stream_encryption;
diff --git a/core/integration/tests/cluster/rolling_sigkill_repro.rs b/core/integration/tests/cluster/rolling_sigkill_repro.rs
new file mode 100644
index 000000000..31a9eca55
--- /dev/null
+++ b/core/integration/tests/cluster/rolling_sigkill_repro.rs
@@ -0,0 +1,320 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! A rolling SIGKILL schedule without Docker: three local server processes, a
+//! persisted topic, producers that retry every batch until it is acked, and
+//! each node killed and restarted in turn under load. Every acked message must
+//! be readable afterwards.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::time::{Duration, Instant};
+
+use iggy::prelude::*;
+use integration::iggy_harness;
+use tokio::sync::Mutex;
+use tokio::time::sleep;
+
+const STREAM_NAME: &str = "repro-stream";
+const TOPIC_NAME: &str = "repro-topic";
+const PARTITIONS: u32 = 6;
+const PRODUCERS: u32 = 6;
+const BATCH: u32 = 20;
+const PAYLOAD_BYTES: usize = 256;
+/// Six producers at this pace send about 2,000 messages per second in total.
+const SEND_INTERVAL: Duration = Duration::from_millis(60);
+const SEND_TIMEOUT: Duration = Duration::from_secs(3);
+const LOAD: Duration = Duration::from_secs(90);
+/// A batch still unacked this long after the load ends is reported, not retried.
+const ACK_GRACE: Duration = Duration::from_secs(180);
+const VERIFY_BUDGET: Duration = Duration::from_secs(120);
+/// Kill and restart times for each node, measured from the start of load.
+const SCHEDULE: [(usize, u64, u64); 3] = [(0, 15, 30), (1, 45, 60), (2, 75, 90)];
+
+type Acked = Arc<Mutex<HashMap<u32, HashSet<String>>>>;
+
+#[derive(Default)]
+struct Counters {
+ send_errors: AtomicUsize,
+ unacked_batches: AtomicUsize,
+ max_latency_ms: AtomicU64,
+}
+
+#[iggy_harness(cluster_nodes = 3)]
+#[ignore = "long-running rolling SIGKILL reproduction"]
+async fn given_load_on_a_persisted_topic_when_every_node_is_sigkilled_in_turn_should_keep_every_acked_message(
+ harness: &mut TestHarness,
+) {
+ let addresses: Vec<String> = harness
+ .all_servers()
+ .iter()
+ .map(|server| server.raw_tcp_addr().expect("tcp address"))
+ .collect();
+ let setup = connect_any(&addresses).await.expect("cluster serves");
+ setup
+ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
+ .await
+ .unwrap();
+ setup
+ .create_topic(
+ &Identifier::named(STREAM_NAME).unwrap(),
+ TOPIC_NAME,
+ &TopicCreateOptions {
+ partitions_count: Some(PARTITIONS),
+ durability: Durability::Persisted,
+ ..TopicCreateOptions::default()
+ },
+ )
+ .await
+ .unwrap();
+ drop(setup);
+
+ let acked: Acked = Arc::default();
+ let counters = Arc::new(Counters::default());
+ let start = Instant::now();
+ // Restarting a node blocks this thread until it boots, so the producers run
+ // on their own runtime and keep writing through every restart.
+ let load = {
+ let (addresses, acked, counters) =
+ (addresses.clone(), Arc::clone(&acked), Arc::clone(&counters));
+ std::thread::spawn(move || {
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(4)
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async move {
+ let producers: Vec<_> = (0..PRODUCERS)
+ .map(|producer| {
+ tokio::spawn(produce(
+ producer,
+ addresses.clone(),
+ Arc::clone(&acked),
+ Arc::clone(&counters),
+ start,
+ ))
+ })
+ .collect();
+ for producer in producers {
+ producer.await.unwrap();
+ }
+ });
+ })
+ };
+
+ for (node, kill_at, restart_at) in SCHEDULE {
+ sleep_until(start, kill_at).await;
+ harness.kill_node(node).expect("SIGKILL node");
+ sleep_until(start, restart_at).await;
+ harness.restart_node(node).expect("restart node");
+ }
+ tokio::task::spawn_blocking(move || load.join().unwrap())
+ .await
+ .unwrap();
+
+ let acked = acked.lock().await;
+ let mut lost = 0usize;
+ let mut total = 0usize;
+ for partition in 0..PARTITIONS {
+ let expected = acked.get(&partition).cloned().unwrap_or_default();
+ total += expected.len();
+ let present = read_partition(&addresses, partition).await;
+ lost += expected.difference(&present).count();
+ }
+ assert_eq!(
+ counters.unacked_batches.load(Ordering::Relaxed),
+ 0,
+ "some batches were never acked (send errors {}, max ack latency {} ms)",
+ counters.send_errors.load(Ordering::Relaxed),
+ counters.max_latency_ms.load(Ordering::Relaxed),
+ );
+ assert_eq!(lost, 0, "{lost} of {total} acked messages are not readable");
+}
+
+async fn sleep_until(start: Instant, at_secs: u64) {
+ let at = Duration::from_secs(at_secs);
+ if let Some(remaining) = at.checked_sub(start.elapsed()) {
+ sleep(remaining).await;
+ }
+}
+
+async fn produce(
+ producer: u32,
+ addresses: Vec<String>,
+ acked: Acked,
+ counters: Arc<Counters>,
+ start: Instant,
+) {
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let topic = Identifier::named(TOPIC_NAME).unwrap();
+ let mut client = None;
+ let mut sequence = 0u64;
+ while start.elapsed() < LOAD {
+ let partition =
+ (producer + u32::try_from(sequence % u64::from(PARTITIONS)).unwrap()) % PARTITIONS;
+ let payloads: Vec<String> = (0..BATCH)
+ .map(|index| {
+ let mut payload = format!("p{producer}-s{sequence}-m{index}-");
+ payload.extend(std::iter::repeat_n('x', PAYLOAD_BYTES - payload.len()));
+ payload
+ })
+ .collect();
+ let sent_at = Instant::now();
+ loop {
+ if start.elapsed() > LOAD + ACK_GRACE {
+ counters.unacked_batches.fetch_add(1, Ordering::Relaxed);
+ return;
+ }
+ if client.is_none() {
+ let first = usize::try_from(producer).unwrap() + usize::try_from(sequence).unwrap();
+ client = connect_any(&rotated(&addresses, first)).await;
+ if client.is_none() {
+ sleep(Duration::from_millis(250)).await;
+ continue;
+ }
+ }
+ let mut messages: Vec<IggyMessage> = payloads
+ .iter()
+ .map(|payload| {
+ IggyMessage::builder()
+ .payload(payload.clone().into())
+ .build()
+ .unwrap()
+ })
+ .collect();
+ let attempt = tokio::time::timeout(
+ SEND_TIMEOUT,
+ client.as_ref().unwrap().send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(partition),
+ &mut messages,
+ ),
+ )
+ .await;
+ match attempt {
+ Ok(Ok(_)) => {
+ let latency = u64::try_from(sent_at.elapsed().as_millis()).unwrap_or(u64::MAX);
+ counters
+ .max_latency_ms
+ .fetch_max(latency, Ordering::Relaxed);
+ acked
+ .lock()
+ .await
+ .entry(partition)
+ .or_default()
+ .extend(payloads);
+ break;
+ }
+ _ => {
+ counters.send_errors.fetch_add(1, Ordering::Relaxed);
+ client = None;
+ sleep(Duration::from_millis(250)).await;
+ }
+ }
+ }
+ sequence += 1;
+ sleep(SEND_INTERVAL).await;
+ }
+}
+
+fn rotated(addresses: &[String], first: usize) -> Vec<String> {
+ let mut rotated = addresses.to_vec();
+ rotated.rotate_left(first % addresses.len());
+ rotated
+}
+
+async fn connect_any(addresses: &[String]) -> Option<IggyClient> {
+ for address in addresses {
+ let config = TcpClientConfig {
+ server_address: address.clone(),
+ ..TcpClientConfig::default()
+ };
+ let Ok(tcp) = TcpClient::create(Arc::new(config)) else {
+ continue;
+ };
+ if Client::connect(&tcp).await.is_err() {
+ continue;
+ }
+ let client = IggyClient::create(ClientWrapper::Tcp(tcp), None, None);
+ if client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .is_ok()
+ {
+ return Some(client);
+ }
+ }
+ None
+}
+
+/// Every payload readable from `partition`, up to the `current_offset` the
+/// topic reports. Empty polls are retried until `VERIFY_BUDGET` runs out.
+async fn read_partition(addresses: &[String], partition: u32) -> HashSet<String> {
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let topic = Identifier::named(TOPIC_NAME).unwrap();
+ let deadline = Instant::now() + VERIFY_BUDGET;
+ let mut present = HashSet::new();
+ let mut next = 0u64;
+ loop {
+ if Instant::now() > deadline {
+ return present;
+ }
+ let Some(client) = connect_any(addresses).await else {
+ sleep(Duration::from_millis(500)).await;
+ continue;
+ };
+ let current = match client.get_topic(&stream, &topic).await {
+ Ok(Some(details)) => details
+ .partitions
+ .iter()
+ .find(|details| details.id == partition)
+ .map(|details| details.current_offset),
+ _ => None,
+ };
+ let Some(current) = current else {
+ sleep(Duration::from_millis(500)).await;
+ continue;
+ };
+ while next <= current && Instant::now() <= deadline {
+ match client
+ .poll_messages(
+ &stream,
+ &topic,
+ Some(partition),
+ &Consumer::default(),
+ &PollingStrategy::offset(next),
+ 1_000,
+ false,
+ )
+ .await
+ {
+ Ok(polled) if !polled.messages.is_empty() => {
+ for message in &polled.messages {
+ present.insert(String::from_utf8_lossy(&message.payload).into_owned());
+ next = next.max(message.header.offset + 1);
+ }
+ }
+ _ => sleep(Duration::from_millis(500)).await,
+ }
+ }
+ if next > current {
+ return present;
+ }
+ }
+}
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 8d40a93b7..2493855b5 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -37,6 +37,7 @@ use crate::state_transfer::{
use crate::types::{
COMMIT_WALK_OPS_MAX, FatalCommit, RepairConclusion, RepairIngest, RepairSession,
};
+use crate::view_fence::{ViewFence, ViewWindowVerdict, view_window_verdict};
use crate::{
AppendResult, Partition, PartitionOffsets, PartitionsConfig, PollFragments, PollQueryResult,
PollingArgs, PollingConsumer,
@@ -119,6 +120,7 @@ pub struct OffsetSpace {
}
// This struct aliases in terms of the code contained the `LocalPartition from `core/server/src/streaming/partitions/local_partition.rs`.
+#[allow(clippy::struct_excessive_bools)]
pub struct IggyPartition<B = IggyMessageBus, SB = PingPongSuperblock>
where
B: MessageBus,
@@ -261,6 +263,18 @@ where
pub(crate) transition: Option<PendingPartitionTransition>,
deferred_views: Vec<server_common::MessageBag>,
pub(crate) pending_consumer_offset_commits: HashMap<u64, PendingConsumerOffsetCommit>,
+ /// Entries journaled before this replica adopted the current view, held back
+ /// from commit and acknowledgement until proven. See [`Self::fence_view_window`].
+ view_fence: Option<ViewFence>,
+ /// Set at boot when the WAL replayed history: the first view adoption after a
+ /// restart fences it even though `log_view` came back unchanged.
+ recovered_unproven: bool,
+ /// Commit point the recovered WAL names. Entries at or below it are
+ /// committed history, so a fence neither holds them back nor proves them.
+ recovered_commit: u64,
+ /// First op of a broken fenced window the I/O lane is truncating. The fence's
+ /// follow-up runs when that truncation lands.
+ unproven_drop_from: Option<u64>,
/// Identity shared with pending polls and replaced when their history retires.
poll_history: PollHistoryId,
incarnation: crate::PartitionIncarnation,
@@ -888,6 +902,10 @@ where
installed_frontier: None,
fatal: None,
pending_consumer_offset_commits: HashMap::new(),
+ view_fence: None,
+ recovered_unproven: false,
+ recovered_commit: 0,
+ unproven_drop_from: None,
pending_commit: None,
local_request_order: Cell::new(0),
no_ack: VecDeque::new(),
@@ -1697,6 +1715,7 @@ where
self.consensus.set_last_prepare_checksum(checksum);
}
self.consensus.restore_commit_state(checkpoint, commit);
+ self.recovered_commit = commit;
}
// Recovery can reuse completed writes whose last barrier was interrupted.
// Include them once before reclaiming any recovered WAL history.
@@ -1750,6 +1769,7 @@ where
}
}
}
+ self.recovered_unproven = persistence.head() > 0;
Ok(())
}
@@ -2316,11 +2336,30 @@ where
return false;
}
self.persist_repaired_prefix();
- let ready = persistence.certify_log_view(
- self.consensus.log_view(),
- self.consensus.sequencer().current_sequence(),
- self.consensus.last_prepare_checksum(),
- );
+ let (op, checksum) = if let Some(fence) = self.view_fence.filter(|_| self.view_fenced()) {
+ // The certificate covers the whole WAL, so while fenced it may only
+ // vouch for a WAL holding nothing above `commit_min`, and only when
+ // the window sits below the suffix a `DoViewChange` reports.
+ let floor = self.consensus.commit_min();
+ if persistence.head() != floor || fence.through >= self.consensus.commit_max() {
+ return false;
+ }
+ let checksum = if floor == 0 {
+ 0
+ } else {
+ let Some(checksum) = persistence.checksum(floor) else {
+ return false;
+ };
+ checksum
+ };
+ (floor, checksum)
+ } else {
+ (
+ self.consensus.sequencer().current_sequence(),
+ self.consensus.last_prepare_checksum(),
+ )
+ };
+ let ready = persistence.certify_log_view(self.consensus.log_view(), op, checksum);
self.start_persistence();
ready
}
@@ -2356,6 +2395,13 @@ where
&prepare.as_slice()[..size_of::<PrepareHeader>()],
)
.map_or(u64::MAX, |header| header.op);
+ // Same cap as `persist_repaired_prefix_through`: a held entry can sit at
+ // the WAL head, and a prepare chaining to the view's entry instead would
+ // fail the WAL's parent check. The proof's release writes the journal's
+ // copy once the window is proven.
+ if self.view_fenced() && op > self.consensus.commit_min() {
+ return false;
+ }
if op > persistence.head().saturating_add(1) {
self.persist_repaired_prefix_through(op.saturating_sub(1));
if op > persistence.head().saturating_add(1) {
@@ -2381,6 +2427,13 @@ where
let Some(persistence) = &self.persistence else {
return;
};
+ // A held entry can sit at the WAL head, where a repaired successor that
+ // chains to the view's entry instead would fail the WAL's parent check.
+ let through = if self.view_fenced() {
+ through.min(self.consensus.commit_min())
+ } else {
+ through
+ };
while let Some(op) = persistence.head().checked_add(1) {
if op > through
|| op > self.consensus.sequencer().current_sequence()
@@ -5826,6 +5879,17 @@ where
self.notify_io();
}
+ /// Run a view adoption's `actions` once the pending truncation lands, so the
+ /// pipeline rebuild reads the journal after the cut.
+ pub fn defer_view_adoption(&mut self, adoption: Vec<consensus::VsrAction>, from: u8) {
+ if let Some(PendingPartitionTransition::Truncate { actions, peer, .. }) =
+ &mut self.transition
+ {
+ actions.extend(adoption);
+ *peer = Some(from);
+ }
+ }
+
#[must_use]
pub const fn view_truncation_pending(&self) -> bool {
matches!(
@@ -6097,7 +6161,19 @@ where
else {
unreachable!("truncate transition exists");
};
- self.remove_resident_range(ops).await?;
+ let from_op = *ops.start();
+ // Another cut that lands first leaves the marked entries resident, so the
+ // marker stays for the proof that finds them again.
+ let unproven_drop = self
+ .unproven_drop_from
+ .is_some_and(|marked| from_op <= marked);
+ if unproven_drop {
+ self.unproven_drop_from = None;
+ }
+ let removed = self.remove_resident_range(ops).await?;
+ if unproven_drop {
+ self.note_unproven_dropped(from_op, removed);
+ }
return Ok(crate::PartitionIoStep::ViewApplied { actions, peer });
}
if matches!(
@@ -8240,6 +8316,18 @@ where
}
return;
}
+ // A Break drops the window from the broken op, inline or on the I/O lane
+ // once a mounted partition's other history work settles. Either way this
+ // prepare would sit above the cut on a head that does not chain to it.
+ // The fence keeps its anchor. The primary resends only ops still short
+ // of quorum, so gap repair, not a retransmit, may be what refills it.
+ if self.consensus().is_follower() && self.offer_view_anchor(&header) {
+ let verdict = self.prove_view_window().await;
+ if matches!(verdict, Some(ViewWindowVerdict::Break { .. })) || self.refuses_prepares() {
+ self.notify_io();
+ return;
+ }
+ }
let journal_holds_op = self.log.journal().inner.holds_op(header.op);
if journal_holds_op {
@@ -8659,6 +8747,15 @@ where
.await;
return;
}
+ // After the in-flight commit: a proof that breaks truncates the journal.
+ if self.view_fenced() {
+ if !self.view_proof_deferred() {
+ self.prove_view_window().await;
+ }
+ if self.view_fenced() && self.recovered_commit <= self.consensus.commit_min() {
+ return;
+ }
+ }
// The primary commits inline via `on_ack` (it drains its own pipeline).
// Backups never populate the pipeline - they journal replicated prepares
@@ -8669,7 +8766,13 @@ where
// tail still finds its headers here (no wedge). Pipeline-first keeps a
// freshly promoted primary (rebuilt pipeline) draining there, avoiding a
// double-count against `advance_commit_min`.
- let mut drained = self.select_persistable_commits(config);
+ // A fenced pipeline holds unproven entries: only the journal walk, capped
+ // at the recovered commit point, may take ops.
+ let mut drained = if self.view_fenced() {
+ Vec::new()
+ } else {
+ self.select_persistable_commits(config)
+ };
let send_client_replies = !drained.is_empty() && self.consensus.is_primary();
if drained.is_empty() {
drained = self.collect_committable_from_journal(COMMIT_WALK_OPS_MAX, config);
@@ -8776,6 +8879,9 @@ where
}
let from_op = self.consensus.commit_min() + 1;
+ if self.view_fenced() && self.recovered_commit < from_op {
+ return Vec::new();
+ }
// Stop below the pipeline head. The drain above holds rather than pops
// when the head is not the op owed next; walking that op out of the
// journal instead advances `commit_min` past a still-resident entry, and
@@ -8794,6 +8900,11 @@ where
.pipeline_head_header()
.filter(|head| head.op >= from_op)
.map_or(commit_max, |head| commit_max.min(head.op - 1));
+ let commit_max = if self.view_fenced() {
+ commit_max.min(self.recovered_commit)
+ } else {
+ commit_max
+ };
let materialize = self.should_persist_messages(config);
self.log
.journal()
@@ -9037,6 +9148,345 @@ where
})
}
+ /// Hold `(fence_floor, through]` back from commit and acknowledgement, where
+ /// `fence_floor` is the higher of `commit_min` and the recovered commit point, until
+ /// [`Self::prove_view_window`] links it into `anchor`, the current view's header
+ /// at `through + 1`. Without an anchor the first current-view prepare at that op
+ /// supplies one (`offer_view_anchor`).
+ ///
+ /// For a replica that adopted a newer view: it can hold an entry from a log the
+ /// view discarded under the view's commit point, which nothing else compares
+ /// against the view's log.
+ pub fn fence_view_window(&mut self, through: u64, anchor: Option<PrepareHeader>) {
+ if through <= self.fence_floor() {
+ self.view_fence = None;
+ return;
+ }
+ let view = self.consensus.view();
+ let previous = self
+ .view_fence
+ .filter(|fence| fence.view == view && fence.through == through);
+ let anchor = anchor
+ .filter(PrepareHeader::anchors_chain)
+ .or_else(|| previous.and_then(|fence| fence.anchor));
+ if previous.is_none() {
+ debug!(
+ namespace_raw = self.namespace().inner(),
+ view,
+ commit_min = self.consensus.commit_min(),
+ floor = self.fence_floor(),
+ recovered_commit = self.recovered_commit,
+ through,
+ anchored = anchor.is_some(),
+ "holding partition entries journaled before this view until its log proves them"
+ );
+ }
+ self.view_fence = Some(ViewFence {
+ view,
+ through,
+ anchor,
+ hole: previous.and_then(|fence| fence.hole),
+ unanchored_ticks: previous.map_or(0, |fence| fence.unanchored_ticks),
+ });
+ }
+
+ /// Highest op through which this replica holds every entry above its commit
+ /// point. Repair progress, since a fenced commit point cannot move until the
+ /// whole window is held.
+ #[must_use]
+ pub fn held_through(&self) -> u64 {
+ self.log
+ .journal()
+ .inner
+ .held_through(self.consensus.commit_min())
+ }
+
+ /// Whether an anchored fence holds every entry of its window, so a proof
+ /// now concludes instead of waiting on repair.
+ #[must_use]
+ pub fn view_fence_held(&self) -> bool {
+ self.view_fence.as_ref().is_some_and(|fence| {
+ fence.anchor.is_some()
+ && fence.through > self.consensus.commit_min()
+ && self
+ .log
+ .journal()
+ .inner
+ .held_through_until(self.consensus.commit_min(), fence.through)
+ >= fence.through
+ })
+ }
+
+ /// Whether a commit walk would run the proof of a held fence now.
+ #[must_use]
+ pub fn view_fence_provable(&self) -> bool {
+ self.fatal.is_none()
+ && !self.materialization_missing
+ && !self.persistence_checkpoint_pending()
+ // The commit walk returns before the proof while the view's log is unproven.
+ && !self.consensus.view_log_unproven()
+ && !self.view_proof_deferred()
+ && self.view_fence_held()
+ }
+
+ /// Whether a proof now would only find the same `Break` again: a Break found
+ /// while history work runs or a transition is queued cuts nothing
+ /// (`drop_unproven_from`), so walking the window rebuilds its header map for
+ /// nothing.
+ fn view_proof_deferred(&self) -> bool {
+ self.history_is_busy() || self.transition.is_some()
+ }
+
+ /// An installed snapshot replaces the history the recovered commit point named.
+ pub(crate) const fn forget_recovered_commit(&mut self) {
+ self.recovered_commit = 0;
+ }
+
+ /// Whether a fenced walk still has recovered committed ops to take.
+ #[must_use]
+ pub fn recovered_prefix_pending(&self) -> bool {
+ self.view_fenced() && self.recovered_commit > self.consensus.commit_min()
+ }
+
+ /// Bottom of a fenced window: what this replica committed, or the recovered
+ /// commit point when the walk has not reached it yet.
+ #[must_use]
+ pub fn fence_floor(&self) -> u64 {
+ self.consensus.commit_min().max(self.recovered_commit)
+ }
+
+ /// Whether a fenced window is still unproven.
+ #[must_use]
+ pub fn view_fenced(&self) -> bool {
+ self.view_fence
+ .as_ref()
+ .is_some_and(|fence| fence.through > self.consensus.commit_min())
+ }
+
+ /// Whether the fence is still waiting for its anchor.
+ #[must_use]
+ pub fn view_fence_unanchored(&self) -> bool {
+ self.view_fenced()
+ && self
+ .view_fence
+ .as_ref()
+ .is_some_and(|fence| fence.anchor.is_none())
+ }
+
+ /// Count one tick without an anchor; true every `every` ticks, when the caller
+ /// should ask the view's primary for its headers again.
+ pub fn view_fence_probe_due(&mut self, every: u32) -> bool {
+ if !self.view_fence_unanchored() {
+ return false;
+ }
+ let Some(fence) = self.view_fence.as_mut() else {
+ return false;
+ };
+ fence.unanchored_ticks = fence.unanchored_ticks.saturating_add(1);
+ if fence.unanchored_ticks < every.max(1) {
+ return false;
+ }
+ fence.unanchored_ticks = 0;
+ true
+ }
+
+ /// Take `header` as the fence's anchor when it is the op above the window and
+ /// was prepared in the fence's view. Only for a header from that view's primary
+ /// (a prepare that passed `replicate_preflight`, or its `StartView`). Returns
+ /// whether the anchor was taken.
+ pub(crate) const fn offer_view_anchor(&mut self, header: &PrepareHeader) -> bool {
+ if let Some(fence) = self.view_fence.as_mut()
+ && fence.anchor.is_none()
+ && fence.view == header.view
+ && header.op == fence.through + 1
+ && header.anchors_chain()
+ {
+ fence.anchor = Some(*header);
+ fence.hole = None;
+ return true;
+ }
+ false
+ }
+
+ /// Lowest op of the fenced window, at or under `commit_max`, with no entry:
+ /// what repair has to fetch before the chain can reach below it.
+ #[must_use]
+ pub fn view_fence_hole(&self) -> Option<u64> {
+ let fence = self.view_fence.as_ref()?;
+ let floor = self.fence_floor();
+ let top = fence.through.min(self.consensus.commit_max());
+ let held = self.log.journal().inner.held_through_until(floor, top);
+ (held < top).then_some(held + 1)
+ }
+
+ /// Residency of a window op. A forced flush evicts past `commit_min` only on a
+ /// caught-up primary or at shutdown, and the flush is capped while fenced, so
+ /// a window op is never ring-only and the index alone answers.
+ fn holds_view_entry(&self, op: u64) -> bool {
+ self.log.journal().inner.holds_op(op)
+ }
+
+ /// Check the fenced window against its anchor. `Proven` lifts the fence, a
+ /// `Break` drops the entries from the broken op so repair refetches the view's,
+ /// a `Hole` waits for repair. `None` while there is no fence or no anchor.
+ ///
+ /// A `Break` cuts the whole suffix from the broken op, entries the anchor
+ /// already proved and current-view prepares included; none of them was acked.
+ /// Without an I/O dispatcher the cut runs here. With one it is handed to the
+ /// I/O lane, or left for the next proof while other history work holds the
+ /// lane, and the entries stay resident until the lane runs it.
+ pub async fn prove_view_window(&mut self) -> Option<ViewWindowVerdict> {
+ let fence = self.view_fence?;
+ let floor = self.fence_floor();
+ if fence.through <= floor {
+ self.view_fence = None;
+ self.persist_repaired_prefix();
+ return Some(ViewWindowVerdict::Proven);
+ }
+ let anchor = fence.anchor?;
+ // A commit floor raised past the hole (a repair floor, an install) took
+ // it out of the window.
+ if let Some(hole) = fence.hole.filter(|hole| *hole > floor)
+ && !self.holds_view_entry(hole)
+ {
+ return Some(ViewWindowVerdict::Hole { op: hole });
+ }
+ let journal = &self.log.journal().inner;
+ let verdict = view_window_verdict(floor, fence.through, anchor.parent, |op| {
+ journal.repair_header(op)
+ });
+ match verdict {
+ ViewWindowVerdict::Proven => {
+ // The WAL flush below reads the last append per op; a duplicate
+ // whose last append is not the entry just proven breaks there.
+ if let Some(op) = (floor + 1..=fence.through).find_map(|op| {
+ let header = journal.repair_header(op)?;
+ let flushed = journal.repair_entry(op)?;
+ let flushed = bytemuck::checked::try_from_bytes::<PrepareHeader>(
+ &flushed.as_slice()[..size_of::<PrepareHeader>()],
+ )
+ .ok()?;
+ (flushed.checksum != header.checksum).then_some(op)
+ }) {
+ self.drop_unproven_from(op).await;
+ return Some(ViewWindowVerdict::Break { op });
+ }
+ debug!(
+ namespace_raw = self.namespace().inner(),
+ through = fence.through,
+ "the adopted view's log proves the held partition entries"
+ );
+ self.view_fence = None;
+ // A Break earlier in this fence left it naming a dropped entry.
+ let head = self.consensus.sequencer().current_sequence();
+ if let Some(header) = self.log.journal().inner.repair_header(head) {
+ self.consensus.set_last_prepare_checksum(header.checksum);
+ }
+ self.persist_repaired_prefix();
+ self.queue_acks_withheld_by_fence(fence.through, head);
+ }
+ ViewWindowVerdict::Hole { op } => {
+ if fence.hole != Some(op) {
+ debug!(
+ namespace_raw = self.namespace().inner(),
+ op, "held partition entries wait on op {op} before they can be proven"
+ );
+ }
+ if let Some(fence) = self.view_fence.as_mut() {
+ fence.hole = Some(op);
+ }
+ }
+ ViewWindowVerdict::Break { op } => self.drop_unproven_from(op).await,
+ }
+ Some(verdict)
+ }
+
+ async fn drop_unproven_from(&mut self, op: u64) {
+ // The evicted ring holds only the applied prefix, so a broken entry above
+ // `commit_min` is always resident.
+ debug_assert!(self.log.journal().inner.holds_op(op));
+ if !self.log.journal().inner.holds_op(op) {
+ error!(
+ namespace_raw = self.namespace().inner(),
+ op,
+ "partition entry at op {op} is not the adopted view's and is already \
+ flushed; marking the partition fatal and shutting down"
+ );
+ self.mark_view_fence_fatal(op);
+ return;
+ }
+ if self.has_io_dispatcher() {
+ // The I/O lane owns a mounted partition's WAL. While other history work
+ // is in flight or a transition is queued the fence keeps holding, and
+ // the next proof finds the same break.
+ if !self.view_proof_deferred() {
+ self.unproven_drop_from = Some(op);
+ self.defer_view_truncation(op..=u64::MAX, Vec::new(), None);
+ }
+ return;
+ }
+ match self.truncate_uncommitted_from(op).await {
+ Ok(removed) => self.note_unproven_dropped(op, removed),
+ Err(error) => {
+ error!(
+ namespace_raw = self.namespace().inner(),
+ op,
+ %error,
+ "cannot drop partition entries from op {op} the adopted view's log does not \
+ chain to; marking the partition fatal and shutting down"
+ );
+ self.mark_view_fence_fatal(op);
+ }
+ }
+ }
+
+ fn note_unproven_dropped(&mut self, op: u64, removed: usize) {
+ warn!(
+ namespace_raw = self.namespace().inner(),
+ op,
+ removed,
+ view = self.consensus.view(),
+ "dropped {removed} partition entries from op {op}: the adopted view's log does not \
+ chain to them"
+ );
+ if let Some(fence) = self.view_fence.as_mut() {
+ fence.hole = None;
+ }
+ self.persist_repaired_prefix();
+ }
+
+ /// Prepares this view's primary sent while the window was fenced were
+ /// journaled but neither persisted nor acked. The primary resends one only
+ /// while it lacks quorum, after its retransmit timeout. Queue their acks for
+ /// the WAL to release as soon as the fence lifts.
+ fn queue_acks_withheld_by_fence(&self, through: u64, head: u64) {
+ let view = self.consensus.view();
+ let mut pending = self.pending_persisted_acks.borrow_mut();
+ for op in through + 1..=head {
+ if let Some(header) = self.log.journal().inner.repair_header(op)
+ && header.view == view
+ {
+ pending.insert(op, header);
+ }
+ }
+ drop(pending);
+ self.notify_io();
+ }
+
+ fn mark_view_fence_fatal(&mut self, op: u64) {
+ self.fatal = Some(FatalCommit {
+ namespace_raw: self.namespace().inner(),
+ op,
+ operation: None,
+ });
+ }
+
+ /// Whether the WAL replayed history at boot, consumed by the first view
+ /// adoption so it fences that history even with `log_view` unchanged.
+ pub fn take_recovered_unproven(&mut self) -> bool {
+ std::mem::take(&mut self.recovered_unproven)
+ }
+
/// Drop an uncommitted view-divergent suffix inline and restore append cursors
/// from the retained prefix. Mounted partitions defer this work to the I/O lane.
///
@@ -9171,8 +9621,27 @@ where
}
}
self.consensus.note_journal_mutation();
+ if *ops.start() <= self.recovered_commit {
+ let lowered = ops.start().saturating_sub(1);
+ error!(
+ namespace_raw = self.namespace().inner(),
+ from_op = *ops.start(),
+ recovered_commit = self.recovered_commit,
+ lowered,
+ "cut partition entries the WAL recovered as committed; lowering the recovered \
+ commit point to {lowered} so repair refetches them"
+ );
+ self.recovered_commit = lowered;
+ }
self.pending_consumer_offset_commits
.retain(|op, _| !ops.contains(op));
+ // Nothing is in flight for a cut range, so the next sweep re-requests it
+ // instead of waiting out the retry timer.
+ if removed > 0
+ && let Some(session) = self.repair.as_mut()
+ {
+ session.requested_from = None;
+ }
self.offset_reservations_need_resync.set(true);
self.truncated_since_scan.set(true);
Ok(removed)
@@ -9282,7 +9751,8 @@ where
}
/// Materialize inline for partitions without a dispatcher. Returns false
- /// while the captured prefix is pending in the WAL.
+ /// while the captured prefix is pending in the WAL. While a view fence is up
+ /// it flushes no further than `commit_min`.
async fn commit_messages_inner(
&mut self,
config: &PartitionsConfig,
@@ -9328,7 +9798,12 @@ where
if self.materialization.is_some() {
return Ok(true);
}
- let through_op = self.consensus.commit_max().min(through_op);
+ let mut through_op = self.consensus.commit_max().min(through_op);
+ // Forced flushes (shutdown, transfer offers) reach here without the walk:
+ // nothing above an unproven window may land in a segment.
+ if self.view_fenced() {
+ through_op = through_op.min(self.consensus.commit_min());
+ }
let prefix_len = self.materializable_prefix_len(through_op);
if prefix_len == 0 {
return Ok(true);
@@ -11928,7 +12403,7 @@ where
}
/// Ingest one repaired prepare: journal + stage it exactly like a live
- /// replicated op, minus the view fence, the gap check, and the ack (the
+ /// replicated op, minus the view-number check, the gap check, and the ack (the
/// op is already committed cluster-wide; there is nobody to ack to). The
/// commit walk runs at `RepairDone`, after the floor is known.
pub async fn apply_repaired_prepare(
@@ -11939,6 +12414,11 @@ where
if self.view_truncation_pending() || !self.prepare_history_available(&header) {
return RepairIngest::Handled;
}
+ // The I/O lane already cut the WAL for a broken fenced window; an entry
+ // journaled now would outlive it there. Refetched once the cut lands.
+ if self.unproven_drop_from.is_some() {
+ return RepairIngest::Handled;
+ }
let Some(session) = self.repair else {
return RepairIngest::Handled;
};
@@ -12342,6 +12822,11 @@ where
if self.fatal.is_some() || self.materialization_missing {
return false;
}
+ // An ack above an unproven window can be wiped when the window turns out
+ // to break; the primary's retransmit re-drives it once the fence lifts.
+ if self.view_fenced() {
+ return false;
+ }
// Durable-before-send: a PrepareOk implies this replica's
// (view, log_view), so it must not leave until they are durable, or a
// crash could recover an older view than the one this ack helped
@@ -29949,6 +30434,7 @@ mod tests {
floor: Some(floor),
peer: 0,
idle_ticks: 0,
+ requested_from: Some(1),
}
}
@@ -29980,58 +30466,738 @@ mod tests {
.expect("journal append");
}
- /// Walking through the head advances `commit_min` past a resident entry only
- /// `on_ack` can pop and answer, after which every later ack finds the drain
- /// empty and no reply is ever shipped.
- #[compio::test]
- async fn given_a_pipeline_head_when_walking_the_journal_should_stop_below_it() {
- let partition = test_partition();
- for op in 1..=4 {
- journal_prepare(&partition, op, Operation::CreateStream).await;
- }
- partition.consensus.restore_commit_state(0, 4);
- partition.consensus.pipeline_message(
- PlaneKind::Partitions,
- &pipeline_prepare(3, Operation::CreateStream),
- );
+ /// A sealed prepare at `op` chained to `parent`, journaled. `request` tells two
+ /// entries at the same op apart.
+ async fn journal_sealed(
+ partition: &IggyPartition<IggyMessageBus>,
+ op: u64,
+ parent: u128,
+ request: u64,
+ ) -> PrepareHeader {
+ let prepare = sealed_prepare(op, parent, request);
+ let header = *prepare.header();
+ partition
+ .log
+ .journal()
+ .inner
+ .append(prepare.into_frozen())
+ .await
+ .expect("journal append");
+ header
+ }
- let ops: Vec<u64> = partition
+ fn sealed_prepare(op: u64, parent: u128, request: u64) -> Message<PrepareHeader> {
+ let size = std::mem::size_of::<PrepareHeader>();
+ Message::<PrepareHeader>::new(size).transmute_header(|_, header: &mut PrepareHeader| {
+ header.command = Command::Prepare;
+ header.client = message_bus::AUTO_COMMIT_CLIENT_ID;
+ header.session = 1;
+ header.request = request;
+ header.op = op;
+ header.parent = parent;
+ header.operation = Operation::CreateStream;
+ header.size = u32::try_from(size).expect("prepare header size fits in u32");
+ header.checksum = header.identity_checksum();
+ })
+ }
+
+ fn walked_ops(partition: &IggyPartition<IggyMessageBus>) -> Vec<u64> {
+ partition
.collect_committable_from_journal(COMMIT_WALK_OPS_MAX, &repair_config())
.into_iter()
- .map(|entry| entry.op)
- .collect();
- assert_eq!(
- ops,
- vec![1, 2],
- "the walk stops at the op the pipeline holds, leaving 3 to on_ack"
- );
+ .map(|header| header.op)
+ .collect()
}
- /// A backup journals replicated prepares and never populates a pipeline, so an
- /// absent head must mean NO ceiling. Read as a ceiling of zero it would stop
- /// every backup's commit walk.
#[compio::test]
- async fn given_an_empty_pipeline_when_walking_the_journal_should_not_cap() {
- let partition = test_partition();
- for op in 1..=3 {
- journal_prepare(&partition, op, Operation::CreateStream).await;
- }
- partition.consensus.restore_commit_state(0, 3);
- assert!(partition.consensus.pipeline_head_header().is_none());
+ async fn given_a_fenced_window_when_the_anchor_proves_it_should_release_the_walk() {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ let second = journal_sealed(&partition, 2, first.checksum, 2).await;
+ partition.consensus.restore_commit_state(0, 2);
+ partition.fence_view_window(2, None);
+ assert!(
+ walked_ops(&partition).is_empty(),
+ "the walk applied an entry the view has not vouched for"
+ );
- let ops: Vec<u64> = partition
- .collect_committable_from_journal(COMMIT_WALK_OPS_MAX, &repair_config())
- .into_iter()
- .map(|entry| entry.op)
- .collect();
- assert_eq!(ops, vec![1, 2, 3], "a backup walks its whole committed run");
+ partition.offer_view_anchor(sealed_prepare(3, second.checksum, 3).header());
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Proven)
+ );
+ assert!(!partition.view_fenced());
+ assert_eq!(walked_ops(&partition), vec![1, 2]);
}
#[compio::test]
- async fn persisted_commit_walk_preserves_pipeline_gaps_and_wal_ceiling() {
- let directory = tempfile::tempdir().unwrap();
- let mut partition = partition_at_view(0, 0);
- partition.set_partition_dir(directory.path().to_string_lossy().into_owned());
+ async fn given_an_old_views_entry_under_the_anchor_when_proving_without_a_dispatcher_should_drop_it_and_stay_fenced()
+ {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ // The view's op 2, which the anchor chains to, is not what is held.
+ let canonical_second = sealed_prepare(2, first.checksum, 2);
+ journal_sealed(&partition, 2, first.checksum, 99).await;
+ partition.consensus.restore_commit_state(0, 2);
+ partition.fence_view_window(
+ 2,
+ Some(*sealed_prepare(3, canonical_second.header().checksum, 3).header()),
+ );
+
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Break { op: 2 })
+ );
+ assert!(!partition.log.journal().inner.holds_op(2));
+ assert!(partition.log.journal().inner.holds_op(1));
+ assert!(
+ walked_ops(&partition).is_empty(),
+ "op 1 is not proven until op 2 is refetched, so the walk must still hold"
+ );
+ assert!(partition.view_fenced());
+ }
+
+ #[compio::test]
+ async fn given_a_hole_in_the_fenced_window_when_proving_should_truncate_nothing() {
+ let mut partition = test_partition();
+ journal_sealed(&partition, 1, 0, 1).await;
+ let third = journal_sealed(&partition, 3, 7, 3).await;
+ partition.consensus.restore_commit_state(0, 3);
+ partition.fence_view_window(3, Some(*sealed_prepare(4, third.checksum, 4).header()));
+
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Hole { op: 2 })
+ );
+ assert!(
+ partition.log.journal().inner.holds_op(1) && partition.log.journal().inner.holds_op(3),
+ "a hole is no evidence against the entries around it"
+ );
+ assert_eq!(partition.view_fence_hole(), Some(2));
+ assert!(partition.view_fenced());
+ }
+
+ #[compio::test]
+ async fn given_a_recovered_commit_point_when_fenced_should_walk_the_committed_prefix() {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ let second = journal_sealed(&partition, 2, first.checksum, 2).await;
+ journal_sealed(&partition, 3, second.checksum, 3).await;
+ partition.consensus.restore_commit_state(0, 3);
+ partition.recovered_commit = 2;
+ partition.fence_view_window(3, None);
+
+ assert!(partition.view_fenced());
+ assert_eq!(
+ walked_ops(&partition),
+ vec![1, 2],
+ "the recovered WAL already named ops 1 and 2 committed"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_wal_naming_a_commit_point_when_restarted_and_fenced_should_commit_the_recovered_prefix()
+ {
+ let directory = tempfile::tempdir().unwrap();
+ let (mut partition, _) = recording_partition_at(1, 3);
+ partition.set_partition_dir(directory.path().to_string_lossy().into_owned());
+ partition.log.retire_front().unwrap();
+ partition
+ .install_empty_segment(&repair_config(), 0)
+ .await
+ .unwrap();
+ let mut journal = journal::PartitionPrepareJournal::open(
+ &directory.path().join("prepares-0"),
+ partition.consensus().group(),
+ 0,
+ )
+ .await
+ .unwrap();
+ let mut parent = 0;
+ for op in 1..=4u64 {
+ let mut prepare = checksummed_segment_prepare(op, parent, op - 1, b"recovered");
+ prepare = prepare.transmute_header(|old, header: &mut PrepareHeader| {
+ *header = old;
+ header.commit = op - 1;
+ header.checksum = header.identity_checksum();
+ });
+ parent = prepare.header().checksum;
+ journal.append(prepare.into_frozen()).await.unwrap();
+ }
+ drop(journal);
+ partition.open_persistence().await.unwrap();
+ assert_eq!(partition.consensus.commit_min(), 0);
+ assert_eq!(partition.consensus.commit_max(), 3);
+
+ partition.fence_view_window(4, None);
+ assert!(partition.view_fenced());
+ partition.commit_journal(&repair_config()).await;
+
+ assert!(partition.fatal.is_none(), "{:?}", partition.fatal);
+ assert_eq!(
+ partition.consensus.commit_min(),
+ 3,
+ "the recovered WAL named ops 1..=3 committed"
+ );
+ assert!(
+ partition.view_fenced(),
+ "op 4 stays held until the view's log proves it"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_fenced_window_when_its_hole_fills_should_report_it_held() {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ partition.consensus.restore_commit_state(0, 3);
+ partition.fence_view_window(3, Some(*sealed_prepare(4, 9, 4).header()));
+ assert_eq!(partition.held_through(), 1);
+ assert!(!partition.view_fence_held());
+
+ let second = journal_sealed(&partition, 2, first.checksum, 2).await;
+ assert!(!partition.view_fence_held());
+ journal_sealed(&partition, 3, second.checksum, 3).await;
+ assert_eq!(partition.held_through(), 3);
+ assert!(partition.view_fence_held());
+ }
+
+ #[compio::test]
+ async fn given_an_anchor_from_another_view_when_offered_should_refuse_it() {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ partition.consensus.restore_commit_state(0, 1);
+ partition.fence_view_window(1, None);
+ let mut other_view = *sealed_prepare(2, first.checksum, 2).header();
+ other_view.view = partition.consensus.view() + 1;
+ assert!(
+ !partition.offer_view_anchor(&other_view),
+ "an anchor from a view the fence was not installed in vouches for the wrong log"
+ );
+ assert!(partition.view_fence_unanchored());
+ }
+
+ #[compio::test]
+ async fn given_an_unsealed_anchor_when_fencing_should_stay_unanchored() {
+ let mut partition = test_partition();
+ journal_sealed(&partition, 1, 0, 1).await;
+ partition.consensus.restore_commit_state(0, 1);
+ // What a `StartView` carries at a commit point the primary holds no
+ // header for: checksum and parent both zero.
+ let blank = PrepareHeader {
+ command: Command::Prepare,
+ op: 2,
+ ..Default::default()
+ };
+ partition.fence_view_window(1, Some(blank));
+ assert!(
+ partition.view_fence_unanchored(),
+ "a blank anchor links any window below it"
+ );
+ assert!(!partition.offer_view_anchor(&blank));
+ }
+
+ /// A Break leaves `last_prepare_checksum` naming the entry it dropped, and the
+ /// refetch fills the op without moving the head; unless the proof refreshes
+ /// it, certification keeps failing on a quiet partition.
+ #[compio::test]
+ async fn given_a_stale_last_prepare_checksum_when_the_window_proves_should_certify() {
+ let (_directory, mut partition, held) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ partition.consensus().set_log_view(view);
+ partition.consensus().restore_commit_state(0, 1);
+ partition
+ .consensus()
+ .set_last_prepare_checksum(held.checksum ^ 1);
+ let anchor = checksummed_segment_prepare(2, held.checksum, 0, b"anchor");
+ partition.fence_view_window(1, Some(*anchor.header()));
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Proven)
+ );
+ partition.ensure_wal_view();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
+ assert_eq!(persistence.certified_log_view(), Some(view));
+ }
+
+ #[compio::test]
+ async fn given_a_fenced_wal_when_certifying_should_vouch_only_for_the_applied_prefix() {
+ let (_directory, mut partition, held) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ partition.consensus().set_log_view(view);
+ partition.consensus().sequencer().set_sequence(4);
+
+ // A window reaching the suffix the next `DoViewChange` reports: muted.
+ partition.consensus().restore_commit_state(held.op, 3);
+ partition.fence_view_window(3, None);
+ assert!(!partition.ensure_wal_view());
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
+ assert_ne!(persistence.certified_log_view(), Some(view));
+
+ // WAL holds nothing above `commit_min`, window below the suffix: certified.
+ partition.consensus().advance_commit_max(4);
+ partition.ensure_wal_view();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
+ assert_eq!(persistence.certified_log_view(), Some(view));
+ }
+
+ #[compio::test]
+ async fn given_an_unanchored_fence_when_ticking_should_ask_for_headers_at_the_pace_given() {
+ let mut partition = test_partition();
+ journal_sealed(&partition, 1, 0, 1).await;
+ partition.consensus.restore_commit_state(0, 1);
+ partition.fence_view_window(1, None);
+ let due: Vec<bool> = (0..6).map(|_| partition.view_fence_probe_due(3)).collect();
+ assert_eq!(due, vec![false, false, true, false, false, true]);
+ }
+
+ #[compio::test]
+ async fn given_a_cached_hole_when_the_missing_op_arrives_should_bypass_the_cache() {
+ let mut partition = test_partition();
+ journal_sealed(&partition, 1, 0, 1).await;
+ let third = journal_sealed(&partition, 3, 7, 3).await;
+ partition.consensus.restore_commit_state(0, 3);
+ partition.fence_view_window(3, Some(*sealed_prepare(4, third.checksum, 4).header()));
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Hole { op: 2 })
+ );
+ assert_eq!(partition.view_fence.and_then(|fence| fence.hole), Some(2));
+ // Op 3 chains to parent 7, which the arriving op 2 is not.
+ journal_sealed(&partition, 2, 0, 99).await;
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Break { op: 2 }),
+ "the cache must be bypassed once the missing op arrives"
+ );
+ }
+
+ /// The fence holds entries back from the WAL too: a stale entry can sit at the
+ /// WAL head, and appending the view's successor on top of it fails the WAL's
+ /// parent check, which fences the whole partition.
+ #[compio::test]
+ async fn given_a_stale_wal_head_when_fenced_should_not_persist_repaired_successors() {
+ let (_directory, mut partition, stale) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ assert_eq!(persistence.head(), stale.op);
+ let canonical = checksummed_segment_prepare(1, 0, 7, b"canonical");
+ let successor =
+ checksummed_segment_prepare(2, canonical.header().checksum, 0, b"successor");
+ partition
+ .log
+ .journal()
+ .inner
+ .append(successor.clone().into_frozen())
+ .await
+ .unwrap();
+ partition.consensus().sequencer().set_sequence(2);
+ partition.consensus().restore_commit_state(0, 2);
+ partition.fence_view_window(1, Some(*successor.header()));
+
+ partition.persist_repaired_prefix();
+ assert_eq!(
+ persistence.head(),
+ stale.op,
+ "a repaired successor reached the WAL above an unproven entry"
+ );
+ assert!(persistence.failure().is_none());
+ }
+
+ /// With an I/O dispatcher the Break is handed to the lane: the stale entry
+ /// stays resident and repair frames are refused until the cut lands, then the
+ /// hole is refetched at once.
+ #[compio::test]
+ async fn given_a_dispatcher_when_the_proof_breaks_should_defer_the_drop_to_the_lane() {
+ let (_directory, mut partition, stale) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ partition.set_io_notifier(
+ Rc::new(|_, _| {}),
+ crate::largest_legal_job_charge().unwrap(),
+ );
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ partition.consensus().set_log_view(view);
+ partition.consensus().restore_commit_state(0, 1);
+ let canonical = checksummed_segment_prepare(1, 0, 7, b"canonical");
+ let anchor = checksummed_segment_prepare(2, canonical.header().checksum, 0, b"anchor");
+ partition.repair = Some(RepairSession {
+ view,
+ ..armed_session(2, 0)
+ });
+ partition.fence_view_window(1, Some(*anchor.header()));
+
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Break { op: 1 })
+ );
+ assert!(
+ partition.log.journal().inner.holds_op(1),
+ "the lane, not the proof, truncates a mounted partition"
+ );
+ assert_eq!(partition.unproven_drop_from, Some(1));
+ assert!(partition.refuses_prepares());
+ partition.apply_repaired_prepare(canonical.clone()).await;
+ partition.apply_repaired_prepare(anchor.clone()).await;
+ assert!(
+ !partition.log.journal().inner.holds_op(2),
+ "a repaired body landed under a pending cut"
+ );
+ assert_eq!(
+ partition
+ .log
+ .journal()
+ .inner
+ .repair_header(1)
+ .map(|header| header.checksum),
+ Some(stale.checksum),
+ "a repaired body journaled before the cut would outlive it in the WAL"
+ );
+
+ let config = repair_config();
+ let mut applied = false;
+ for _ in 0..64 {
+ match partition.resume_io(&config).await {
+ crate::PartitionIoStep::Ready(plan) => {
+ execute_partition_io_plan(&mut partition, &config, plan).await;
+ }
+ crate::PartitionIoStep::Progress => {}
+ crate::PartitionIoStep::ViewApplied { .. } => {
+ applied = true;
+ break;
+ }
+ crate::PartitionIoStep::Pending => {
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
+ }
+ _ => panic!("unexpected partition I/O continuation"),
+ }
+ }
+ assert!(applied, "the lane never ran the truncation");
+ assert!(!partition.log.journal().inner.holds_op(1));
+ assert_eq!(partition.unproven_drop_from, None);
+ assert_eq!(persistence.head(), 0);
+ assert_eq!(
+ partition.repair.map(|session| session.requested_from),
+ Some(None),
+ "the hole must be refetched without waiting out the retry timer"
+ );
+ assert!(partition.view_fenced());
+ assert!(partition.fatal.is_none());
+ }
+
+ /// A proof that breaks while other history work holds the lane drops
+ /// nothing yet; the anchor that triggered it must not reach the WAL on top
+ /// of the stale head it does not chain to. Its retransmit skips the proof,
+ /// since the fence already holds the anchor, so the WAL itself has to refuse.
+ #[compio::test]
+ async fn given_busy_history_when_an_anchor_breaks_the_window_should_not_append_it_to_the_wal() {
+ let (_directory, mut partition, _stale) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ partition.set_io_notifier(
+ Rc::new(|_, _| {}),
+ crate::largest_legal_job_charge().unwrap(),
+ );
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ partition.consensus().set_log_view(view);
+ partition.consensus().restore_commit_state(0, 1);
+ partition.fence_view_window(1, None);
+ partition.maintenance.flush_through = Some(1);
+ assert!(partition.history_is_busy());
+ let canonical = checksummed_segment_prepare(1, 0, 7, b"canonical");
+ let anchor = checksummed_segment_prepare(2, canonical.header().checksum, 0, b"anchor")
+ .transmute_header(|mut original, header: &mut PrepareHeader| {
+ original.view = view;
+ original.checksum = original.identity_checksum();
+ *header = original;
+ });
+ let sequence = partition.consensus().sequencer().current_sequence();
+ partition.on_replicate(anchor.clone()).await;
+ assert!(!partition.view_fence_unanchored());
+ assert!(partition.view_fenced());
+ assert_eq!(partition.unproven_drop_from, None);
+ assert!(
+ !partition.log.journal().inner.holds_op(2),
+ "the anchor was journaled above a window its proof broke"
+ );
+ assert_eq!(
+ partition.consensus().sequencer().current_sequence(),
+ sequence
+ );
+ partition.on_replicate(anchor).await;
+ let _ = persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await;
+ assert!(
+ persistence.failure().is_none(),
+ "the anchor was appended on a WAL head it does not chain to: {:?}",
+ persistence.failure()
+ );
+ assert_eq!(persistence.head(), 1);
+ assert!(partition.fatal.is_none());
+ }
+
+ /// The lane's cut is pending but nothing else refuses the frame: the gate
+ /// on the pending drop alone keeps a repaired body out of the journal.
+ #[compio::test]
+ async fn given_a_pending_unproven_drop_when_a_repair_frame_arrives_should_not_journal_it() {
+ let mut partition = test_partition();
+ partition.repair = Some(armed_session(3, 0));
+ partition.unproven_drop_from = Some(1);
+ assert!(!partition.refuses_prepares());
+
+ partition
+ .apply_repaired_prepare(repaired_send_prepare(1, 0, 1))
+ .await;
+ assert_eq!(
+ partition.log.journal().inner.resident_count(),
+ 0,
+ "a repaired body was journaled under a pending cut"
+ );
+ assert_eq!(partition.consensus().sequencer().current_sequence(), 0);
+ }
+
+ /// A repair commit floor (`complete_repair` taking the peer's
+ /// `RangeEvicted` floor) can raise the fence floor past the hole the last
+ /// proof stopped at. The cached hole then names an op outside the window
+ /// and must not keep the proof waiting on it.
+ #[compio::test]
+ async fn given_a_cached_hole_under_a_raised_floor_when_proving_should_not_wait_on_it() {
+ let mut partition = test_partition();
+ journal_sealed(&partition, 1, 0, 1).await;
+ let third = journal_sealed(&partition, 3, 7, 3).await;
+ partition.consensus.restore_commit_state(0, 3);
+ partition.fence_view_window(3, Some(*sealed_prepare(4, third.checksum, 4).header()));
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Hole { op: 2 })
+ );
+
+ partition.consensus.set_commit_floor(2);
+ assert_eq!(partition.view_fence_hole(), None);
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Proven),
+ "the proof waited on a hole below the raised floor"
+ );
+ assert!(!partition.view_fenced());
+ }
+
+ /// A prepare received while the window was fenced is journaled but neither
+ /// persisted nor acked. The primary resends it only while it lacks quorum,
+ /// after a retransmit timeout, so proving the window releases its ack
+ /// without waiting for one.
+ #[compio::test]
+ async fn given_a_prepare_received_while_fenced_when_the_window_is_proven_should_ack_it_without_a_retransmit()
+ {
+ let (_directory, mut partition, held) = partition_with_pending_durable_ack().await;
+ let persistence = partition.persistence.as_ref().unwrap().clone();
+ let sent = partition.consensus().message_bus().sent_to_replicas.clone();
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ partition.consensus().set_log_view(view);
+ partition.consensus().restore_commit_state(0, 1);
+ // The held op 1 was journaled behind the append counter's back; wake
+ // the counter so the live successor continues it at offset 1.
+ partition.offset_space.append_live = true;
+ partition.dirty_offset.store(0, Ordering::Relaxed);
+ let successor = checksummed_segment_prepare(2, held.checksum, 1, b"successor")
+ .transmute_header(|mut original, header: &mut PrepareHeader| {
+ original.view = view;
+ original.checksum = original.identity_checksum();
+ *header = original;
+ });
+ partition.fence_view_window(1, Some(*successor.header()));
+ partition.on_replicate(successor).await;
+ assert!(partition.log.journal().inner.holds_op(2));
+ assert_eq!(
+ persistence.head(),
+ 1,
+ "a fenced replica persisted a prepare above the window"
+ );
+ sent.borrow_mut().clear();
+
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Proven)
+ );
+ for _ in 0..4 {
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
+ partition.drive_persistence().await;
+ }
+ let acked: Vec<u64> = sent
+ .borrow()
+ .iter()
+ .filter_map(|(_, frame)| {
+ let header =
+ bytemuck::checked::try_from_bytes::<PrepareOkHeader>(frame.as_slice()).ok()?;
+ (header.command == Command::PrepareOk).then_some(header.op)
+ })
+ .collect();
+ assert!(
+ acked.contains(&2),
+ "the fence lifted without acking op 2: {acked:?}"
+ );
+ assert_eq!(persistence.head(), 2);
+ assert!(partition.pending_persisted_acks.borrow().is_empty());
+ assert!(partition.fatal.is_none());
+ }
+
+ /// Only a prepare of the view that proved the window was withheld by the
+ /// fence. An older view's entry above it is not the view's, and its ack
+ /// would vouch for a log the view never sent.
+ #[compio::test]
+ async fn given_an_older_views_entry_above_the_window_when_proven_should_not_queue_its_ack() {
+ let mut partition = test_partition();
+ let first = journal_sealed(&partition, 1, 0, 1).await;
+ let view = partition.consensus().view() + 1;
+ partition.consensus.set_view(view);
+ let in_view = |op: u64, parent: u128| {
+ sealed_prepare(op, parent, op).transmute_header(
+ |mut original, header: &mut PrepareHeader| {
+ original.view = view;
+ original.checksum = original.identity_checksum();
+ *header = original;
+ },
+ )
+ };
+ let anchor = in_view(2, first.checksum);
+ let older = journal_sealed(&partition, 2, first.checksum, 2).await;
+ let current = in_view(3, older.checksum);
+ partition
+ .log
+ .journal()
+ .inner
+ .append(current.into_frozen())
+ .await
+ .expect("journal append");
+ partition.consensus().sequencer().set_sequence(3);
+ partition.consensus.restore_commit_state(0, 1);
+ partition.fence_view_window(1, Some(*anchor.header()));
+
+ assert_eq!(
+ partition.prove_view_window().await,
+ Some(ViewWindowVerdict::Proven)
+ );
+ let queued: Vec<u64> = partition
+ .pending_persisted_acks
+ .borrow()
+ .keys()
+ .copied()
+ .collect();
+ assert_eq!(queued, vec![3], "an older view's entry had its ack queued");
+ }
+
+ #[compio::test]
+ async fn given_a_fenced_window_when_acking_should_withhold_the_ack() {
+ let (mut partition, _) = recording_partition_at(1, 3);
+ let prepare = sealed_prepare(1, 0, 1);
+ let first = *prepare.header();
+ partition
+ .log
+ .journal()
+ .inner
+ .append(prepare.into_frozen())
+ .await
+ .expect("journal append");
+ partition.consensus.sequencer().set_sequence(1);
+ partition.consensus.restore_commit_state(0, 1);
+ assert!(
+ partition.send_prepare_ok(&first).await,
+ "an unfenced backup acks what it holds"
+ );
+ partition.fence_view_window(1, None);
+ assert!(
+ !partition.send_prepare_ok(&first).await,
+ "an ack above an unproven window can be wiped when the window breaks"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_staged_offset_under_commit_max_when_its_entry_is_truncated_should_drop_the_staging()
+ {
+ let mut partition = test_partition();
+ journal_prepare(&partition, 1, Operation::StoreConsumerOffset).await;
+ journal_prepare(&partition, 2, Operation::StoreConsumerOffset).await;
+ partition.consensus.sequencer().set_sequence(2);
+ partition.consensus.restore_commit_state(0, 2);
+ partition.stage_consumer_offset_upsert(2, ConsumerKind::Consumer, 8, 0, false);
+ partition.truncate_uncommitted_from(2).await.unwrap();
+ assert!(
+ !partition.pending_consumer_offset_commits.contains_key(&2),
+ "the stale cursor would be applied in place of the view's op 2"
+ );
+ }
+
+ /// Walking through the head advances `commit_min` past a resident entry only
+ /// `on_ack` can pop and answer, after which every later ack finds the drain
+ /// empty and no reply is ever shipped.
+ #[compio::test]
+ async fn given_a_pipeline_head_when_walking_the_journal_should_stop_below_it() {
+ let partition = test_partition();
+ for op in 1..=4 {
+ journal_prepare(&partition, op, Operation::CreateStream).await;
+ }
+ partition.consensus.restore_commit_state(0, 4);
+ partition.consensus.pipeline_message(
+ PlaneKind::Partitions,
+ &pipeline_prepare(3, Operation::CreateStream),
+ );
+
+ let ops: Vec<u64> = partition
+ .collect_committable_from_journal(COMMIT_WALK_OPS_MAX, &repair_config())
+ .into_iter()
+ .map(|entry| entry.op)
+ .collect();
+ assert_eq!(
+ ops,
+ vec![1, 2],
+ "the walk stops at the op the pipeline holds, leaving 3 to on_ack"
+ );
+ }
+
+ /// A backup journals replicated prepares and never populates a pipeline, so an
+ /// absent head must mean NO ceiling. Read as a ceiling of zero it would stop
+ /// every backup's commit walk.
+ #[compio::test]
+ async fn given_an_empty_pipeline_when_walking_the_journal_should_not_cap() {
+ let partition = test_partition();
+ for op in 1..=3 {
+ journal_prepare(&partition, op, Operation::CreateStream).await;
+ }
+ partition.consensus.restore_commit_state(0, 3);
+ assert!(partition.consensus.pipeline_head_header().is_none());
+
+ let ops: Vec<u64> = partition
+ .collect_committable_from_journal(COMMIT_WALK_OPS_MAX, &repair_config())
+ .into_iter()
+ .map(|entry| entry.op)
+ .collect();
+ assert_eq!(ops, vec![1, 2, 3], "a backup walks its whole committed run");
+ }
+
+ #[compio::test]
+ async fn persisted_commit_walk_preserves_pipeline_gaps_and_wal_ceiling() {
+ let directory = tempfile::tempdir().unwrap();
+ let mut partition = partition_at_view(0, 0);
+ partition.set_partition_dir(directory.path().to_string_lossy().into_owned());
partition.runtime_options.durability = iggy_common::Durability::Persisted;
partition.open_persistence().await.unwrap();
let persistence = Rc::clone(partition.persistence.as_ref().unwrap());
@@ -32276,6 +33442,42 @@ mod tests {
);
}
+ #[compio::test]
+ async fn given_a_fenced_window_when_flushing_should_write_nothing_above_commit_min() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let log_path = dir.path().join("segment.log");
+ let index_path = dir.path().join("segment.index");
+ let mut fixture = PersistFixture::new(
+ log_path.to_str().expect("utf-8 path"),
+ index_path.to_str().expect("utf-8 path"),
+ )
+ .await;
+ let partition = &mut fixture.partition;
+ partition.log.journal().inner.set_repair_retention(true);
+ partition.repair = Some(armed_session(3, 0));
+ for op in 1..=3 {
+ partition
+ .apply_repaired_prepare(repaired_send_prepare(
+ op,
+ u128::from(op - 1),
+ u128::from(op),
+ ))
+ .await;
+ }
+ partition.consensus().advance_commit_max(3);
+ partition.fence_view_window(3, None);
+
+ partition
+ .flush_committed_messages(&repair_config())
+ .await
+ .expect("flush");
+ assert!(
+ std::fs::read(&log_path).unwrap().is_empty(),
+ "a forced flush wrote unproven entries to the segment"
+ );
+ assert!((1..=3).all(|op| partition.log.journal().inner.holds_op(op)));
+ }
+
#[compio::test]
async fn owned_materialization_keeps_physical_cursors_private_until_acceptance() {
let directory = tempfile::tempdir().unwrap();
diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs
index 03bbd5060..56d418ac5 100644
--- a/core/partitions/src/journal.rs
+++ b/core/partitions/src/journal.rs
@@ -203,6 +203,10 @@ where
/// A resident op follows a higher one, so the next op to flush can sit
/// behind the front. Flushes wait for `sort_by_op`.
out_of_order: Cell<bool>,
+ /// `(floor, through)`: every op in `floor + 1..=through` is resident. Repair
+ /// ticks resume the walk from here instead of rescanning the held prefix, and
+ /// any removal resets it.
+ held_run: Cell<(u64, u64)>,
}
/// How many evicted entries each partition retains for repair. Sized to
@@ -237,6 +241,7 @@ where
resident_control_ops: Cell::new(0),
evicted_through: Cell::new(0),
out_of_order: Cell::new(false),
+ held_run: Cell::new((0, 0)),
}
}
}
@@ -819,6 +824,7 @@ where
resident_control_ops: Cell::new(0),
evicted_through: Cell::new(0),
out_of_order: Cell::new(false),
+ held_run: Cell::new((0, 0)),
}
}
@@ -828,6 +834,7 @@ where
self.first_header_by_op.borrow_mut().clear();
unsafe { &mut *self.op_to_storage_offset.get() }.clear();
unsafe { &mut *self.offset_to_op.get() }.clear();
+ self.held_run.set((0, 0));
unsafe { &mut *self.timestamp_to_op.get() }.clear();
self.resident_control_ops.set(0);
self.out_of_order.set(false);
@@ -882,6 +889,41 @@ where
op_to_storage_offset.contains_key(&op)
}
+ /// Highest op `h >= floor` such that every op in `floor + 1..=h` is resident.
+ pub fn held_through(&self, floor: u64) -> u64 {
+ self.held_through_until(floor, u64::MAX)
+ }
+
+ /// [`Self::held_through`] capped at `top`, walking only the resident ops at
+ /// or below it.
+ pub fn held_through_until(&self, floor: u64, top: u64) -> u64 {
+ if floor >= top {
+ return floor;
+ }
+ let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() };
+ let (run_floor, run_through) = self.held_run.get();
+ let mut through = if run_floor <= floor && run_through > floor {
+ run_through.min(top)
+ } else {
+ floor
+ };
+ if through < top {
+ for op in op_to_storage_offset
+ .range(through + 1..=top)
+ .map(|(op, _)| *op)
+ {
+ if op != through + 1 {
+ break;
+ }
+ through = op;
+ }
+ }
+ if through > floor && (run_floor > floor || run_through < through) {
+ self.held_run.set((floor, through));
+ }
+ through
+ }
+
/// Whether any op in `ops` is resident, in one pass over the headers: the
/// range can be far wider than what is resident.
pub fn holds_op_in(&self, ops: RangeInclusive<u64>) -> bool {
@@ -1589,6 +1631,23 @@ mod tests {
assert_eq!(journal.last_op(), Some(3));
}
+ #[compio::test]
+ async fn given_a_run_past_the_top_when_capping_held_through_should_stop_at_the_top() {
+ let journal = PartitionJournal::<PartitionJournalMemStorage>::default();
+ for op in [1, 2, 3, 4, 6] {
+ journal
+ .append(build_prepare(op, HEADER_SIZE + 16).into_frozen())
+ .await
+ .expect("append");
+ }
+
+ assert_eq!(journal.held_through(0), 4);
+ assert_eq!(journal.held_through_until(0, 2), 2);
+ assert_eq!(journal.held_through_until(0, 6), 4);
+ assert_eq!(journal.held_through_until(3, 3), 3);
+ assert_eq!(journal.held_through_until(5, 3), 5);
+ }
+
#[compio::test]
async fn repaired_window_shape_rejects_unbounded_sparse_window_before_allocation() {
let journal = PartitionJournal::<PartitionJournalMemStorage>::default();
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs
index 29268f238..67c10a625 100644
--- a/core/partitions/src/lib.rs
+++ b/core/partitions/src/lib.rs
@@ -44,6 +44,7 @@ mod segment_recovery;
pub mod state_transfer;
mod stream_keyring;
mod types;
+mod view_fence;
#[cfg(feature = "simulator")]
mod wal_storage;
#[cfg(feature = "simulator")]
@@ -70,6 +71,7 @@ pub use io::{
largest_legal_job_charge,
};
pub use journal::{EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY};
+pub use view_fence::ViewWindowVerdict;
/// Offsets a partition claims in its superblock ahead of the mint counter
/// before it will append, so a crash-restarted replica resumes above every
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs
index 3c50ca345..51280950d 100644
--- a/core/partitions/src/state_transfer.rs
+++ b/core/partitions/src/state_transfer.rs
@@ -4262,6 +4262,7 @@ where
self.owner_install_barrier = 0;
self.observed_view = self.consensus().view();
self.repair = None;
+ self.forget_recovered_commit();
self.transfer_offer_cache.borrow_mut().take();
}
diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs
index aa703101b..144ce8503 100644
--- a/core/partitions/src/types.rs
+++ b/core/partitions/src/types.rs
@@ -308,9 +308,17 @@ pub struct RepairSession {
pub floor: Option<u64>,
/// The peer serving this stream (re-request target on stall).
pub peer: u8,
- /// Ticks since the stream last made progress; at
- /// [`REPAIR_RETRY_TICKS`] the remaining window is re-requested.
+ /// Ticks since the stream last made progress. At [`REPAIR_RETRY_TICKS`],
+ /// or at once while [`Self::requested_from`] is `None`, the remaining
+ /// window is re-requested and the count restarts.
pub idle_ticks: u32,
+ /// First op of the latest request. The next chunk is pulled once the held
+ /// run reaches it, which works while a fenced commit point cannot move.
+ ///
+ /// `None` after any removal of resident entries, so the next sweep asks for
+ /// the cut window from the held run without waiting out
+ /// [`REPAIR_RETRY_TICKS`].
+ pub requested_from: Option<u64>,
}
/// How a repair-window commit walk concluded, decided by
diff --git a/core/partitions/src/view_fence.rs b/core/partitions/src/view_fence.rs
new file mode 100644
index 000000000..7e4b3c44b
--- /dev/null
+++ b/core/partitions/src/view_fence.rs
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Entries a replica journaled before adopting a newer view, held back from
+//! commit and acknowledgement until the view's log proves them.
+
+use iggy_binary_protocol::{CHECKSUM_UNSEALED, PrepareHeader};
+
+/// An unproven window `(floor, through]` installed in `view` and, once known, that
+/// view's header at `through + 1`, whose `parent` the window must chain into. The
+/// floor is `IggyPartition::fence_floor`: the higher of `commit_min` and the
+/// commit point recovered from the WAL, which the fenced walk may still reach.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ViewFence {
+ pub view: u32,
+ pub through: u64,
+ pub anchor: Option<PrepareHeader>,
+ /// The op the last proof stopped at for lack of an entry. Until it arrives
+ /// nothing below it can change, so a proof does not rescan the window.
+ pub hole: Option<u64>,
+ /// Ticks spent without an anchor, paced by [`crate::IggyPartition::view_fence_probe_due`].
+ pub unanchored_ticks: u32,
+}
+
+/// What the `parent` chain says about a fenced window.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ViewWindowVerdict {
+ /// Every op in the window links into the anchor.
+ Proven,
+ /// No entry at `op`, so nothing at or below it can be linked yet.
+ Hole { op: u64 },
+ /// The entry at `op` belongs to a log the view discarded: it is not the one
+ /// the op above chains to, or it is not the copy the WAL flush would write.
+ Break { op: u64 },
+}
+
+/// Walk `(floor, through]` top-down from `anchor_parent`, the checksum the view's
+/// entry at `through` must carry.
+///
+/// An unsealed entry breaks the chain: it names no identity to compare, and its
+/// zero `parent` would link every op below it, as [`PrepareHeader::anchors_chain`]
+/// refuses for the anchor.
+#[must_use]
+pub fn view_window_verdict(
+ floor: u64,
+ through: u64,
+ anchor_parent: u128,
+ header_at: impl Fn(u64) -> Option<PrepareHeader>,
+) -> ViewWindowVerdict {
+ let mut expected = anchor_parent;
+ for op in (floor + 1..=through).rev() {
+ let Some(local) = header_at(op) else {
+ return ViewWindowVerdict::Hole { op };
+ };
+ if local.checksum == CHECKSUM_UNSEALED || local.checksum != expected {
+ return ViewWindowVerdict::Break { op };
+ }
+ expected = local.parent;
+ }
+ ViewWindowVerdict::Proven
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{ViewWindowVerdict, view_window_verdict};
+ use iggy_binary_protocol::{Command, Operation, PrepareHeader};
+ use std::collections::BTreeMap;
+
+ const FLOOR: u64 = 3;
+ const THROUGH: u64 = 6;
+
+ fn sealed(op: u64, parent: u128, request: u64) -> PrepareHeader {
+ let mut header = PrepareHeader {
+ command: Command::Prepare,
+ operation: Operation::SendMessages,
+ op,
+ parent,
+ request,
+ ..Default::default()
+ };
+ header.checksum = header.identity_checksum();
+ header
+ }
+
+ /// The view's log over `FLOOR + 1 ..= THROUGH + 1`, each entry chained to the
+ /// one below. The last entry is the anchor.
+ fn view_log() -> BTreeMap<u64, PrepareHeader> {
+ let mut parent = 0;
+ (FLOOR + 1..=THROUGH + 1)
+ .map(|op| {
+ let header = sealed(op, parent, op);
+ parent = header.checksum;
+ (op, header)
+ })
+ .collect()
+ }
+
+ fn verdict(log: &BTreeMap<u64, PrepareHeader>) -> ViewWindowVerdict {
+ let anchor = log[&(THROUGH + 1)];
+ view_window_verdict(FLOOR, THROUGH, anchor.parent, |op| log.get(&op).copied())
+ }
+
+ #[test]
+ fn given_the_views_own_entries_when_proving_should_report_proven() {
+ assert_eq!(verdict(&view_log()), ViewWindowVerdict::Proven);
+ }
+
+ #[test]
+ fn given_an_old_views_entry_under_the_anchor_when_proving_should_report_its_op() {
+ let mut log = view_log();
+ // Same op, different request: what an isolated old primary journals.
+ log.insert(FLOOR + 1, sealed(FLOOR + 1, 0, 99));
+ assert_eq!(
+ verdict(&log),
+ ViewWindowVerdict::Break { op: FLOOR + 1 },
+ "a stale entry under the anchor was accepted as the view's"
+ );
+ }
+
+ #[test]
+ fn given_a_hole_under_the_anchor_when_proving_should_report_the_hole_not_a_break() {
+ let mut log = view_log();
+ log.remove(&(FLOOR + 2));
+ log.insert(FLOOR + 1, sealed(FLOOR + 1, 0, 99));
+ assert_eq!(
+ verdict(&log),
+ ViewWindowVerdict::Hole { op: FLOOR + 2 },
+ "a hole is missing data, not evidence against the entry below it"
+ );
+ }
+
+ #[test]
+ fn given_an_unsealed_entry_in_the_window_when_proving_should_report_a_break() {
+ let mut log = view_log();
+ // Unsealed, with the zero parent an unsealed producer leaves: accepting it
+ // would link every op below it unchecked.
+ let mut unsealed = log[&(FLOOR + 2)];
+ unsealed.checksum = 0;
+ unsealed.parent = 0;
+ log.insert(FLOOR + 2, unsealed);
+ log.insert(FLOOR + 1, sealed(FLOOR + 1, 0, 99));
+ assert_eq!(verdict(&log), ViewWindowVerdict::Break { op: FLOOR + 2 });
+ }
+
+ #[test]
+ fn given_an_empty_window_when_proving_should_report_proven() {
+ assert_eq!(
+ view_window_verdict(THROUGH, THROUGH, 0, |_| None),
+ ViewWindowVerdict::Proven
+ );
+ }
+}
diff --git a/core/server/config.toml b/core/server/config.toml
index b207f8f2a..ac9f94fdb 100644
--- a/core/server/config.toml
+++ b/core/server/config.toml
@@ -546,6 +546,9 @@ view_probe_attempts_max = 5
# on a single dropped frame. Paces both the metadata and partition repair loops;
# must be nonzero. Paces streams that are already OPEN only: how long a hole
# waits before one is opened for it is repair_gap_debounce_interval below.
+# Also paces the RequestStartView probe a partition backup sends while its
+# entries from before a newer view wait for a header to prove them against;
+# those probes have no attempt cap.
repair_retry_interval = "1s"
# How long a backup must hold committed ops it cannot walk to before the shard's
@@ -566,7 +569,7 @@ repair_retry_interval = "1s"
# per node, so its detector is not rate-capped.
repair_gap_debounce_interval = "1s"
-# Prepares a peer serves per repair round before the requester walks to the next
+# Prepares a peer serves per repair round before the requester asks for the next
# chunk (integer). Each frame rides the per-peer message-bus queue, so this must
# stay strictly below message_bus.peer_queue_capacity or a full round overruns
# the queue and drops frames. Must be > 0 and <= 1024.
diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs
index d46083229..ea23d14ec 100644
--- a/core/server/src/partition_reconciler.rs
+++ b/core/server/src/partition_reconciler.rs
@@ -4027,6 +4027,7 @@ mod tests {
floor: None,
peer: PEER,
idle_ticks: 0,
+ requested_from: Some(1),
});
for op in 1..=8 {
let parent = partition.consensus().last_prepare_checksum();
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 7ecdcc0e1..2e7322d7f 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -35,10 +35,10 @@ pub use router::CONSENSUS_TICK_INTERVAL;
#[cfg(feature = "simulator")]
use consensus::LocalPipeline;
use consensus::{
- ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind,
- DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane,
- PlaneKind, RepairEvidence, STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES,
- Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus,
+ AdoptedView, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX,
+ DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle,
+ Pipeline, Plane, PlaneKind, RepairEvidence, STATE_TRANSFER_MAX_DECODE_RETRIES,
+ STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus,
build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, encode_prepare_headers,
fatal, repaired_frontier_update, repaired_prepare_evidence, restamp_prepare_view,
verify_prepare_integrity,
@@ -1067,8 +1067,8 @@ impl ShardFrame {
///
/// The per-peer bus queues are bounded (`peer_queue_capacity`)
/// and overrun frames drop silently, so an unbounded burst loses its own tail;
-/// the receiver pulls the window chunk by chunk instead (each walked
-/// `RepairDone` immediately requests the next chunk while progress holds).
+/// the receiver pulls the window chunk by chunk instead (each `RepairDone`
+/// whose held run reached the requested chunk asks for the next).
///
/// Runtime default; the server overrides the live ceiling per shard from
/// `[cluster] repair_chunk_max` at bootstrap.
@@ -5152,6 +5152,7 @@ where
return;
};
partition.ensure_materialization_recovery();
+ let prior_log_view = partition.consensus().log_view();
let actions = match partition.consensus().try_handle_start_view(
PlaneKind::Partitions,
&header,
@@ -5193,7 +5194,15 @@ where
);
return;
}
- if !partition.consensus().is_transferring() && partition.consensus().view_log_unproven() {
+ let adopted = !actions.is_empty();
+ let transferring = partition.consensus().is_transferring();
+ // Consumed only by an adoption that fences, so a transfer that aborts still
+ // leaves the recovered suffix to the next one.
+ let recovered = adopted && !transferring && partition.take_recovered_unproven();
+ let fence_below = adopted
+ && !transferring
+ && (recovered || prior_log_view < partition.consensus().view());
+ if !transferring && (partition.consensus().view_log_unproven() || fence_below) {
// Any stream armed before this adoption belongs to the superseded
// view. Repair bodies carry no nonce, so drop the receiving session
// before reconciling or arming the new view's canonical range.
@@ -5203,6 +5212,9 @@ where
// like it, pending-less adoptions (empty StartView suffix) still sweep
// the relics above the adopted head.
let pending = partition.consensus().pending_view_log();
+ if fence_below {
+ fence_view_window_below(partition, pending.as_ref());
+ }
if partition.has_io_dispatcher()
&& let Some(from_op) = partition_view_divergence_from(partition, pending.as_ref())
{
@@ -5210,12 +5222,26 @@ where
return;
}
reconcile_partition_view_divergence(self.id, partition, pending.as_ref()).await;
+ if fence_below {
+ let truncating = partition.view_truncation_pending();
+ partition.prove_view_window().await;
+ // A Break queued a cut, and the pipeline rebuild must read the journal after it.
+ if !truncating && partition.view_truncation_pending() {
+ partition.defer_view_adoption(actions, header.replica);
+ return;
+ }
+ }
if partition.consensus().view_log_unproven() {
self.maybe_request_partition_repair(partition, header.replica)
.await;
return;
}
}
+ anchor_view_fence_from_start_view(self.id, partition, &header, suffix_body).await;
+ if partition.view_truncation_pending() {
+ partition.defer_view_adoption(actions, header.replica);
+ return;
+ }
self.finish_partition_view_adoption(
IggyNamespace::from_raw(header.group),
header.replica,
@@ -5608,6 +5634,7 @@ where
// by the backup gap check (adoption already advanced its sequencer to
// the head), so repair is the only channel that can deliver them.
let to_op = repair_serve_ceiling(&header, partition.consensus());
+ let fence_floor = partition.view_fenced().then(|| partition.fence_floor());
// `None` means the journal holds NOTHING, not "nothing was evicted":
// the partition journal is memory-only and `clear_all` wipes the
// evicted ring with it, so a freshly installed or freshly restarted
@@ -5649,6 +5676,14 @@ where
if !partition.consensus().view_log_entry_proven(op) {
break;
}
+ let Some(held) = partition.log.journal().inner.repair_header(op) else {
+ break;
+ };
+ if !fenced_serve_admits(fence_floor, op, || {
+ partition.consensus().adopted_view_vouches_for(&held)
+ }) {
+ break;
+ }
let Some(entry) = partition.log.journal().inner.repair_entry(op) else {
break;
};
@@ -5953,6 +5988,7 @@ where
if closes_proof_gap {
let pending = partition.consensus().pending_view_log();
reconcile_partition_view_divergence(self.id, partition, pending.as_ref()).await;
+ anchor_view_fence_from_view_log(partition, pending.as_ref()).await;
}
if ingest == RepairIngest::Unvouched {
tracing::debug!(
@@ -5962,6 +5998,8 @@ where
"refusing a repaired partition prepare the view log does not vouch for"
);
self.retarget_partition_repair(partition).await;
+ } else if closes_proof_gap {
+ self.request_moved_proof_op(partition).await;
}
}
@@ -6229,8 +6267,9 @@ where
// op (the peer's served-through claim proves nothing about
// delivery on a lossy bus). Proof moves backward through gaps,
// then may need a higher canonical header's body. Either change
- // pulls the next chunk without waiting for the retry timer.
- let before = partition.consensus().commit_min();
+ // pulls the next chunk without waiting for the retry timer, and so
+ // does a held run that reached the chunk just requested; a shorter
+ // run is left to the retry timer.
if let partitions::RepairConclusion::FloorRefused { floor, to_op } =
partition.complete_repair(&config).await
{
@@ -6290,13 +6329,18 @@ where
"partition journal repair complete"
);
} else {
- let commit_min = partition.consensus().commit_min();
+ let repaired_through = partition.held_through();
let proof_op = partition.consensus().view_log_repair_op();
let next = partition.repair.as_mut().and_then(|live| {
let from_op = proof_op.filter(|&op| op != live.from_op).or_else(|| {
- partition_repair_next_chunk(before, commit_min, live.fetch_to_op)
+ partition_repair_next_chunk(
+ live.requested_from,
+ repaired_through,
+ live.fetch_to_op,
+ )
})?;
live.from_op = from_op;
+ live.requested_from = Some(from_op);
Some((live.peer, live.nonce, from_op, live.fetch_to_op))
});
if let Some((peer, nonce, from_op, to_op)) = next {
@@ -6489,6 +6533,14 @@ where
// already holds a header for, so the scan would report a gap nothing
// fills. Backups reach this on StartView adoption; a primary-elect has no
// adoption to hang it off.
+ // `log_view` still names the old log here: it is raised only when the
+ // view starts.
+ let recovered = partition.take_recovered_unproven();
+ let fence_below =
+ recovered || partition.consensus().log_view() < partition.consensus().view();
+ if fence_below {
+ fence_view_window_below(partition, Some(&pending));
+ }
if partition.has_io_dispatcher()
&& let Some(from_op) = partition_view_divergence_from(partition, Some(&pending))
{
@@ -6496,6 +6548,9 @@ where
return;
}
reconcile_partition_view_divergence(self.id, partition, Some(&pending)).await;
+ if fence_below {
+ partition.prove_view_window().await;
+ }
if partition.view_truncation_pending()
|| (partition.consensus().view_log_unproven()
&& partition.consensus().view_log_repair_op().is_none())
@@ -6550,6 +6605,11 @@ where
.await;
return;
}
+ // Everything is held, but not yet proven to be the view's: starting
+ // now commits whatever this replica journaled in an older view.
+ if partition.view_fenced() {
+ return;
+ }
let actions = consensus.start_pending_view(PlaneKind::Partitions);
let (local_actions, wire_actions) = split_local_actions(actions);
@@ -8394,6 +8454,19 @@ where
walk_cursor.get_or_insert(namespace);
}
}
+ // Entries that filled the window after its last proof re-drive
+ // nothing themselves once there is nothing left to fetch. A backup
+ // in a view change must not truncate what its DVC reported.
+ if consensus::repair_session_live(partition.consensus())
+ && partition.view_fence_provable()
+ {
+ if walks < PARTITION_WALKS_PER_TICK_MAX {
+ walks += 1;
+ partition.commit_journal(partitions.config()).await;
+ } else {
+ walk_cursor.get_or_insert(namespace);
+ }
+ }
if let Some(fault) = partition.fatal() {
if fatal.is_none() {
fatal = Some(fault.clone());
@@ -8440,26 +8513,32 @@ where
);
continue;
}
- let repair_from = partition
- .consensus()
- .view_log_repair_op()
- .unwrap_or_else(|| commit_min.saturating_add(1));
+ let proof_op = partition.consensus().view_log_repair_op();
+ let journal = &partition.log.journal().inner;
let due = partition.repair.as_mut().and_then(|session| {
if !session_live {
return None;
}
session.idle_ticks += 1;
- if session.idle_ticks < repair_retry_ticks {
+ if !partition_repair_retry_due(
+ session.requested_from,
+ session.idle_ticks,
+ repair_retry_ticks,
+ ) {
return None;
}
session.idle_ticks = 0;
- session.from_op = repair_from;
- Some((
- session.peer,
- session.nonce,
- repair_from,
- session.fetch_to_op,
- ))
+ // A pending proof names the hole to fetch. Otherwise the held
+ // run, not `commit_min`, marks progress: a fenced commit point
+ // cannot move until the whole window is held.
+ let from_op = proof_op
+ .unwrap_or_else(|| journal.held_through(commit_min).saturating_add(1));
+ session.from_op = from_op;
+ session.requested_from = Some(from_op);
+ if proof_op.is_none() && from_op > session.fetch_to_op {
+ return None;
+ }
+ Some((session.peer, session.nonce, from_op, session.fetch_to_op))
});
// A session pins its peer and fences every arming site while it
// stands, so a peer that cannot answer wedges the group harder
@@ -8607,7 +8686,11 @@ where
continue;
};
let probe = partition_gap_probe(partition);
- let walk_stalled = !partition.has_pending_commit() && group_is_walk_stalled(&probe);
+ // A fenced walk takes only the recovered committed prefix; the
+ // proof, re-driven by repair and the anchor, releases the rest.
+ let walk_stalled = !partition.has_pending_commit()
+ && group_is_walk_stalled(&probe)
+ && (!partition.view_fenced() || partition.recovered_prefix_pending());
// The RATE cap only. The concurrency cap lives in the arm fn,
// which is the funnel every arming site goes through; resolved
// before the debounce either way, so a refusal keeps the group
@@ -8709,6 +8792,27 @@ where
}
}
+ // A fence with no anchor stays shut until the view's primary names the
+ // header above it; ask again rather than wait for the next prepare.
+ if let Some(partition) = partitions.get_mut_by_ns(&namespace)
+ && !partition
+ .consensus()
+ .is_primary_for_view(partition.consensus().view())
+ // The answer would re-adopt the view and drop the live repair session.
+ && !partition.consensus().view_log_unproven()
+ && partition.view_fence_probe_due(self.repair_retry_ticks.get())
+ {
+ // A probe promises nothing about this replica's view, so it skips the
+ // superblock gate, which cannot certify while the fence holds the
+ // WAL back.
+ let consensus = partition.consensus();
+ let probe = [VsrAction::SendRequestStartView {
+ view: consensus.view(),
+ group: consensus.group(),
+ }];
+ dispatch_vsr_actions::<B, _, MJ>(consensus, None, &probe).await;
+ }
+
// Transfer stall retry: descriptor and chunk frames are
// fire-and-forget, so a lost one must not wedge the session (and
// the rejoin behind it) forever. Budget-bounded: a peer that died
@@ -9816,6 +9920,7 @@ where
floor: None,
peer,
idle_ticks: 0,
+ requested_from: Some(from_op),
});
tracing::info!(
shard = self.id,
@@ -9839,6 +9944,44 @@ where
true
}
+ /// Request the op the view-log proof names once it climbed above the one the
+ /// live session last asked for. The walk runs downward, so it climbs only after
+ /// everything below is held and nothing of that request is still in flight.
+ /// `RepairDone` pulls the same op, but a body that lands after its `RepairDone`
+ /// moves the proof when nothing is left to notice, and the op would wait out
+ /// the retry timer.
+ #[allow(clippy::future_not_send)]
+ async fn request_moved_proof_op(&self, partition: &mut IggyPartition<B, SB>)
+ where
+ B: MessageBus,
+ {
+ let Some(proof_op) = partition.consensus().view_log_repair_op() else {
+ return;
+ };
+ let next = partition
+ .repair
+ .as_mut()
+ .filter(|live| proof_op > live.from_op)
+ .map(|live| {
+ live.from_op = proof_op;
+ live.requested_from = Some(proof_op);
+ (live.peer, live.nonce, live.fetch_to_op)
+ });
+ let Some((peer, nonce, to_op)) = next else {
+ return;
+ };
+ self.send_request_prepares(
+ partition.consensus(),
+ peer,
+ nonce,
+ proof_op,
+ to_op,
+ partition.consensus().group(),
+ partition.created_revision(),
+ )
+ .await;
+ }
+
/// Partition twin of [`Self::retarget_metadata_repair`]: re-request the
/// window from the view's primary after a repaired prepare the view log does
/// not vouch for.
@@ -9951,6 +10094,7 @@ where
floor: None,
peer,
idle_ticks: 0,
+ requested_from: Some(from_op),
});
tracing::info!(
shard = self.id,
@@ -11609,6 +11753,12 @@ fn repair_op_in_scope(
})
}
+/// A fenced replica serves what it has proven: ops up to `fence_floor`, and above
+/// it only entries the adopted view's log names, which `build_dvc_suffix` advertises.
+fn fenced_serve_admits(fence_floor: Option<u64>, op: u64, vouched: impl FnOnce() -> bool) -> bool {
+ fence_floor.is_none_or(|floor| op <= floor || vouched())
+}
+
/// Ceiling on the op range a repair request may ask this replica to walk.
///
/// Only peers in this view may receive an uncommitted suffix. During view
@@ -11947,15 +12097,33 @@ fn partition_repair_fetch_to_op(
}
/// Start of the next chunk to pull after a `RepairDone`, or `None` when the
-/// walk made no progress (the stall retry owns the remainder) or already
-/// stands at the session's fetch ceiling. The sweep closes such a session,
-/// because `fetch_to_op` never sits below `commit_to_op`.
+/// held run did not reach the chunk just requested (the stall retry owns the
+/// remainder) or already stands at the session's fetch ceiling. The sweep
+/// closes such a session, because `fetch_to_op` never sits below
+/// `commit_to_op`.
///
/// A session can outlive its last fetchable op until that close, and
/// `from_op > to_op` fails `RequestPreparesHeader::validate` on the serving
/// peer, which drops the frame as unparsable.
-fn partition_repair_next_chunk(before: u64, commit_min: u64, fetch_to_op: u64) -> Option<u64> {
- (commit_min > before && commit_min < fetch_to_op).then_some(commit_min + 1)
+fn partition_repair_next_chunk(
+ requested_from: Option<u64>,
+ held_through: u64,
+ fetch_to_op: u64,
+) -> Option<u64> {
+ (requested_from.is_none_or(|from_op| held_through >= from_op) && held_through < fetch_to_op)
+ .then_some(held_through + 1)
+}
+
+/// Whether the stall sweep re-requests a session's remaining window this tick.
+/// A session whose held entries a proof just dropped has nothing in flight for
+/// the cut window, so waiting out the retry timer only stalls the fenced
+/// replica's catch-up.
+const fn partition_repair_retry_due(
+ requested_from: Option<u64>,
+ idle_ticks: u32,
+ retry_ticks: u32,
+) -> bool {
+ requested_from.is_none() || idle_ticks >= retry_ticks
}
/// Highest adopted suffix op whose bodies are not all present above `commit_max`.
@@ -12029,7 +12197,10 @@ where
.log
.journal()
.inner
- .holds_op(commit_min.saturating_add(1));
+ .holds_op(commit_min.saturating_add(1))
+ // The fenced commit walk cannot reach a hole above the next op, so only
+ // repair fills it.
+ && partition.view_fence_hole().is_none();
// Same discipline, one guard deeper: the suffix test walks the header vec,
// so it runs only for a group that HAS an unfinished suffix and already
// owes nothing else.
@@ -12437,15 +12608,17 @@ where
{
return None;
}
- let applied_floor = partition
- .consensus()
- .commit_min()
+ // This cut stays above the recovered commit point, which the WAL names committed.
+ let committed_floor = partition
+ .fence_floor()
.max(partition.log.journal().inner.evicted_through());
let op_head = pending.map_or_else(
|| partition.consensus().sequencer().current_sequence(),
|pending| pending.op_head,
);
- let above_head = op_head.max(applied_floor).checked_add(1)?;
+ // An adopted head never omits a known commit (`handle_start_view`).
+ debug_assert!(op_head >= partition.fence_floor());
+ let above_head = op_head.max(committed_floor).checked_add(1)?;
partition
.log
.journal()
@@ -12533,6 +12706,179 @@ async fn reconcile_partition_view_divergence<B, SB>(
}
}
+/// The lowest header in a view's suffix that can anchor a fence. A blank slot
+/// names no identity, so its zero `parent` would link anything below it.
+fn view_fence_anchor(headers: &[PrepareHeader]) -> Option<PrepareHeader> {
+ headers
+ .iter()
+ .filter(|header| {
+ matches!(dvc_header_kind(header), DvcHeaderKind::Valid) && header.anchors_chain()
+ })
+ .min_by_key(|header| header.op)
+ .copied()
+}
+
+/// Fence what this replica holds between its commit point and the view's lowest
+/// canonical header (or the adopted head, when the `StartView` named none).
+///
+/// Those entries can come from a log the view discarded, and nothing else
+/// compares them before the commit walk applies them. Fencing, not truncating: a
+/// hole or a missing anchor is no evidence against an entry, and truncating on
+/// either drops entries this replica may be the only one to hold. An empty window
+/// is fenced too, so whatever repair puts there is proven before it is applied.
+fn fence_view_window_below<B, SB>(partition: &mut IggyPartition<B, SB>, pending: Option<&MergedLog>)
+where
+ B: MessageBus,
+ SB: journal::superblock::SuperblockStore,
+{
+ let anchor = pending.and_then(|pending| view_fence_anchor(&pending.headers));
+ // Without an anchor the window runs to the adopted head, which the pending log
+ // names; the sequencer can sit above it after an above-head truncation.
+ let through = match (anchor, pending) {
+ (Some(anchor), _) => anchor.op.saturating_sub(1),
+ (None, Some(pending)) => pending.op_head,
+ (None, None) => partition.consensus().sequencer().current_sequence(),
+ };
+ partition.fence_view_window(through, anchor);
+}
+
+/// Anchor a fence that adopted a blank head once a proven repair body fills that
+/// head in the view's log, as a `StartView` naming the header would have.
+#[allow(clippy::future_not_send)]
+async fn anchor_view_fence_from_view_log<B, SB>(
+ partition: &mut IggyPartition<B, SB>,
+ pending: Option<&MergedLog>,
+) where
+ B: MessageBus,
+ SB: journal::superblock::SuperblockStore,
+{
+ if !partition.view_fence_unanchored()
+ || pending.is_none_or(|pending| view_fence_anchor(&pending.headers).is_none())
+ {
+ return;
+ }
+ fence_view_window_below(partition, pending);
+ partition.prove_view_window().await;
+}
+
+/// From this view's primary, and not behind what this replica already holds: an
+/// older frame would truncate above its own head.
+const fn start_view_may_anchor(
+ (frame_view, sender, frame_op): (u32, u8, u64),
+ view: u32,
+ primary: u8,
+ head: u64,
+) -> bool {
+ frame_view == view && sender == primary && frame_op >= head
+}
+
+/// Hand a fence still waiting for its anchor the headers a same-view `StartView`
+/// carries: the probe answer a fenced replica asked for. A held entry the answer
+/// names differently, or a proof that breaks, queues a truncation, as on adoption.
+#[allow(clippy::future_not_send)]
+async fn anchor_view_fence_from_start_view<B, SB>(
+ shard: u16,
+ partition: &mut IggyPartition<B, SB>,
+ header: &StartViewHeader,
+ suffix_body: &[u8],
+) where
+ B: MessageBus,
+ SB: journal::superblock::SuperblockStore,
+{
+ let consensus = partition.consensus();
+ if !partition.view_fence_unanchored()
+ || !start_view_may_anchor(
+ (header.view, header.replica, header.op),
+ consensus.view(),
+ consensus.primary_index(header.view),
+ consensus.sequencer().current_sequence(),
+ )
+ {
+ return;
+ }
+ let pending = match MergedLog::from_start_view(header, suffix_body) {
+ Ok(pending) if !pending.headers.is_empty() => pending,
+ Ok(_) => return,
+ Err(error) => {
+ tracing::warn!(
+ shard,
+ namespace_raw = consensus.group(),
+ view = header.view,
+ %error,
+ "start_view suffix did not decode, so the fence keeps waiting for an anchor"
+ );
+ return;
+ }
+ };
+ let proof = AdoptedView {
+ view: header.view,
+ group: consensus.group(),
+ partition_incarnation: consensus.partition_incarnation(),
+ op_head: pending.op_head,
+ commit_max: pending.commit_max,
+ headers: pending.headers,
+ };
+ if let Err(error) = proof.validate(consensus.cluster(), consensus.view()) {
+ tracing::warn!(
+ shard,
+ namespace_raw = consensus.group(),
+ view = header.view,
+ %error,
+ "start_view suffix names another partition or incarnation, so the fence keeps waiting for an anchor"
+ );
+ return;
+ }
+ let pending = proof.merged_log();
+ if let Some(op) = first_view_divergence(partition, &pending) {
+ tracing::warn!(
+ shard,
+ namespace_raw = partition.consensus().group(),
+ op,
+ view = header.view,
+ "held partition entry at op {op} disagrees with the view's log, dropping from there"
+ );
+ partition.consensus().fence_view_log(op, op);
+ if partition.has_io_dispatcher() {
+ partition.defer_view_truncation(op..=u64::MAX, Vec::new(), Some(header.replica));
+ } else if let Err(error) = partition.truncate_uncommitted_from(op).await {
+ tracing::error!(shard, op, %error, "could not truncate diverging partition entries");
+ } else {
+ reconcile_partition_view_divergence(shard, partition, Some(&pending)).await;
+ }
+ }
+ fence_view_window_below(partition, Some(&pending));
+ partition.prove_view_window().await;
+}
+
+/// The lowest op above the fence floor where this replica holds an entry the view's
+/// log names differently. The fence's chain proof stops below its anchor, so held
+/// entries at or above it are compared here.
+fn first_view_divergence<B, SB>(
+ partition: &IggyPartition<B, SB>,
+ pending: &MergedLog,
+) -> Option<u64>
+where
+ B: MessageBus,
+ SB: journal::superblock::SuperblockStore,
+{
+ let journal = &partition.log.journal().inner;
+ let floor = partition.fence_floor();
+ let top = pending.op_head.min(journal.last_op()?);
+ pending
+ .headers
+ .iter()
+ .filter(|canonical| {
+ canonical.op > floor
+ && canonical.op <= top
+ && matches!(dvc_header_kind(canonical), DvcHeaderKind::Valid)
+ && journal
+ .header_by_op(canonical.op)
+ .is_some_and(|local| !header_is_view_entry(&local, canonical))
+ })
+ .map(|canonical| canonical.op)
+ .min()
+}
+
/// Whether a locally journaled header IS the entry the view's log names at that op.
///
/// Identity, not presence: otherwise a stale prepare at the right op reads as
@@ -13265,13 +13611,27 @@ mod repair_scope_tests {
#[test]
fn given_a_walk_at_the_fetch_ceiling_when_repair_done_lands_should_not_request_a_chunk() {
- assert_eq!(super::partition_repair_next_chunk(4, 7, 8), Some(8));
- // No progress leaves the remainder to the stall retry.
- assert_eq!(super::partition_repair_next_chunk(7, 7, 8), None);
+ assert_eq!(super::partition_repair_next_chunk(Some(4), 7, 8), Some(8));
+ // A held run short of the requested chunk leaves the remainder to the
+ // stall retry, so a peer with nothing new to serve is not re-asked at once.
+ assert_eq!(super::partition_repair_next_chunk(Some(8), 7, 9), None);
// Progress that reached the ceiling has nothing left to ask for: the
// sweep closes the session, and `9..=8` is not a range.
- assert_eq!(super::partition_repair_next_chunk(7, 8, 8), None);
- assert_eq!(super::partition_repair_next_chunk(7, 9, 8), None);
+ assert_eq!(super::partition_repair_next_chunk(Some(7), 8, 8), None);
+ assert_eq!(super::partition_repair_next_chunk(Some(7), 9, 8), None);
+ }
+
+ #[test]
+ fn given_a_cut_window_when_repair_done_lands_should_pull_from_the_held_run() {
+ assert_eq!(super::partition_repair_next_chunk(None, 2, 8), Some(3));
+ assert_eq!(super::partition_repair_next_chunk(None, 8, 8), None);
+ }
+
+ #[test]
+ fn given_a_cut_window_when_the_sweep_runs_should_re_request_before_the_retry_timer() {
+ assert!(super::partition_repair_retry_due(None, 1, 10));
+ assert!(!super::partition_repair_retry_due(Some(3), 9, 10));
+ assert!(super::partition_repair_retry_due(Some(3), 10, 10));
}
#[test]
@@ -13714,6 +14074,73 @@ mod view_coverage_tests {
}
}
+#[cfg(test)]
+mod view_fence_gate_tests {
+ //! The gates around the partition view fence, apart from the partition.
+
+ use super::{fenced_serve_admits, start_view_may_anchor, view_fence_anchor};
+ use consensus::dvc_blank;
+ use iggy_binary_protocol::{Command, Operation, PrepareHeader};
+
+ fn sealed(op: u64, parent: u128) -> PrepareHeader {
+ let mut header = PrepareHeader {
+ command: Command::Prepare,
+ operation: Operation::SendMessages,
+ op,
+ parent,
+ ..Default::default()
+ };
+ header.checksum = header.identity_checksum();
+ header
+ }
+
+ #[test]
+ fn given_a_blank_commit_point_when_choosing_an_anchor_should_take_the_lowest_sealed_header() {
+ let above = sealed(6, 7);
+ let headers = [sealed(7, above.checksum), above, dvc_blank(5)];
+ assert_eq!(
+ view_fence_anchor(&headers).map(|anchor| anchor.op),
+ Some(6),
+ "a blank's zero parent links any window below it"
+ );
+ assert_eq!(view_fence_anchor(&[dvc_blank(5)]), None);
+ }
+
+ #[test]
+ fn given_op_one_when_choosing_an_anchor_should_accept_its_zero_parent() {
+ assert_eq!(
+ view_fence_anchor(&[sealed(1, 0)]).map(|anchor| anchor.op),
+ Some(1)
+ );
+ assert_eq!(
+ view_fence_anchor(&[sealed(2, 0)]),
+ None,
+ "a zero parent above op 1 vouches for nothing"
+ );
+ }
+
+ #[test]
+ fn given_a_start_view_when_deciding_whether_it_anchors_should_require_this_views_primary_at_or_above_the_head()
+ {
+ let start_view = |view: u32, replica: u8, op: u64| (view, replica, op);
+ assert!(start_view_may_anchor(start_view(2, 1, 9), 2, 1, 9));
+ assert!(!start_view_may_anchor(start_view(1, 1, 9), 2, 1, 9));
+ assert!(!start_view_may_anchor(start_view(2, 0, 9), 2, 1, 9));
+ assert!(
+ !start_view_may_anchor(start_view(2, 1, 8), 2, 1, 9),
+ "an older frame truncates above its own head"
+ );
+ }
+
+ #[test]
+ fn given_a_fenced_server_when_serving_should_admit_only_proven_entries_above_its_floor() {
+ assert!(fenced_serve_admits(None, 9, || false));
+ assert!(fenced_serve_admits(Some(4), 4, || false));
+ assert!(!fenced_serve_admits(Some(4), 5, || false));
+ assert!(fenced_serve_admits(Some(4), 5, || true));
+ }
+}
+
#[cfg(test)]
mod dvc_suffix_window_tests {
//! The suffix window's floor is a scan bound, not a commit point.
diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs
index 59045b6d4..5e900a154 100644
--- a/core/shard/src/partition_io.rs
+++ b/core/shard/src/partition_io.rs
@@ -2490,6 +2490,7 @@ mod tests {
floor: None,
peer: 1,
idle_ticks: 0,
+ requested_from: Some(1),
});
let reply =
Message::<RepairRangeReplyHeader>::new(size_of::<RepairRangeReplyHeader>())
@@ -2562,6 +2563,7 @@ mod tests {
floor: None,
peer: 0,
idle_ticks: 0,
+ requested_from: Some(1),
});
assert_eq!(
partition.complete_repair(partitions.config()).await,
@@ -3209,6 +3211,7 @@ mod tests {
floor: None,
peer: 0,
idle_ticks: 0,
+ requested_from: Some(1),
});
owner.plane.partitions().insert(namespace, partition);
@@ -5297,6 +5300,7 @@ mod tests {
owner.on_repair_prepare(repaired).await;
}
let before = partition.consensus().commit_min();
+ let requested = requests.borrow().len();
for command in [Command::RangeEvicted, Command::RepairDone] {
let foreign_reply = reply.clone().transmute_header(
|header, foreign: &mut RepairRangeReplyHeader| {
@@ -5309,8 +5313,9 @@ mod tests {
owner.on_repair_range_reply(&foreign_reply).await;
assert_eq!(partition.consensus().commit_min(), before);
assert_eq!(partition.repair.unwrap().floor, None);
- assert!(
- requests.borrow().is_empty(),
+ assert_eq!(
+ requests.borrow().len(),
+ requested,
"a different peer cannot complete or advance this repair session"
);
}
@@ -7465,6 +7470,173 @@ mod tests {
}
}
+ #[compio::test]
+ async fn given_a_blank_start_view_head_when_repair_arrives_should_preserve_and_prove_history() {
+ const HEAD: u64 = 3;
+ for held in [false, true] {
+ for canonical in [true, false] {
+ let (bus, _probes) = probe_capturing_bus();
+ let (owner, _sender) = test_owner(&bus, None);
+ let namespace = IggyNamespace::new(0, 0, 0);
+ let prepares =
+ Box::pin(blank_head_backup(&owner, &bus, namespace, HEAD, held)).await;
+ let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap();
+ // Bodies stamped in an older view prove nothing, but stay held.
+ assert!(partition.consensus().view_log_unproven());
+ assert!((1..=HEAD).all(|op| partition.log.journal().inner.holds_op(op)));
+ assert!(partition.view_fence_unanchored());
+ partition
+ .commit_journal(owner.plane.partitions().config())
+ .await;
+ assert_eq!(partition.consensus().commit_min(), 0);
+
+ let head = *prepares.last().unwrap().header();
+ let answer = if canonical { head } else { forked(head) };
+ owner
+ .on_start_view(start_view_suffix(
+ namespace,
+ BLANK_HEAD_VIEW,
+ HEAD,
+ HEAD,
+ &[answer],
+ ))
+ .await;
+ let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap();
+ partition
+ .commit_journal(owner.plane.partitions().config())
+ .await;
+ if canonical {
+ assert!(!partition.view_fenced());
+ assert_eq!(partition.consensus().commit_min(), HEAD);
+ } else {
+ assert!(
+ partition.consensus().commit_min() < HEAD,
+ "held={held}: a head the view names differently must not commit"
+ );
+ }
+ }
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_probe_answer_naming_a_held_entry_differently_when_proving_should_not_commit_it()
+ {
+ const HEAD: u64 = 4;
+ let (bus, _probes) = probe_capturing_bus();
+ let (owner, _sender) = test_owner(&bus, None);
+ let namespace = IggyNamespace::new(0, 0, 0);
+ let prepares = Box::pin(blank_head_backup(&owner, &bus, namespace, HEAD, true)).await;
+ let headers: Vec<_> = prepares.iter().map(|prepare| *prepare.header()).collect();
+ // The lowest sealed header anchors the fence below the diverging head.
+ let answer = [forked(headers[3]), headers[2], headers[1]];
+ owner
+ .on_start_view(start_view_suffix(
+ namespace,
+ BLANK_HEAD_VIEW,
+ HEAD,
+ HEAD,
+ &answer,
+ ))
+ .await;
+ let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap();
+ partition
+ .commit_journal(owner.plane.partitions().config())
+ .await;
+ assert!(partition.consensus().commit_min() < HEAD);
+ assert!(
+ partition.view_truncation_pending() || !partition.log.journal().inner.holds_op(HEAD)
+ );
+ }
+
+ const BLANK_HEAD_VIEW: u32 = 3;
+
+ /// A bus that records the `RequestStartView` probes sent to the view's primary.
+ fn probe_capturing_bus() -> (Rc<IggyMessageBus>, Rc<RefCell<Vec<RequestStartViewHeader>>>) {
+ let bus = Rc::new(IggyMessageBus::new(0));
+ let probes = Rc::new(RefCell::new(Vec::new()));
+ let captured = Rc::clone(&probes);
+ bus.set_replica_forward_fn(Box::new(move |_, _, frame| {
+ let message =
+ Message::<GenericHeader>::try_from(Owned::copy_from_slice(frame.as_slice()))
+ .unwrap();
+ if message.header().command == Command::RequestStartView {
+ let probe = Message::<RequestStartViewHeader>::try_from(Owned::copy_from_slice(
+ frame.as_slice(),
+ ))
+ .unwrap();
+ captured.borrow_mut().push(*probe.header());
+ }
+ Ok(())
+ }));
+ let primary = u8::try_from(BLANK_HEAD_VIEW % 3).unwrap();
+ assert!(bus.owner_table().try_claim(primary, 1));
+ (bus, probes)
+ }
+
+ /// A backup that adopted [`BLANK_HEAD_VIEW`] from a `StartView` with a blank head
+ /// and then received the history up to `head` through repair, in bodies stamped
+ /// by an older view that the view's log does not vouch for. With `journaled`, it
+ /// held that history before the adoption.
+ #[allow(clippy::future_not_send)]
+ async fn blank_head_backup(
+ owner: &IoTestShard,
+ bus: &Rc<IggyMessageBus>,
+ namespace: IggyNamespace,
+ head: u64,
+ journaled: bool,
+ ) -> Vec<Message<PrepareHeader>> {
+ let prepares = primary_prepares(bus, namespace, head).await;
+ let mut partition =
+ repairing_replica(bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0);
+ partition.repair = None;
+ if journaled {
+ for prepare in &prepares {
+ partition.on_replicate(prepare.clone()).await;
+ }
+ }
+ owner.plane.partitions().remove(&namespace).unwrap();
+ owner.plane.partitions().insert(namespace, partition);
+ owner
+ .on_start_view(start_view_suffix(
+ namespace,
+ BLANK_HEAD_VIEW,
+ head,
+ head,
+ &[consensus::dvc_blank(head)],
+ ))
+ .await;
+ let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap();
+ assert!(partition.consensus().view_log_unproven());
+ assert!(!partition.view_truncation_pending());
+ assert_eq!(partition.log.journal().inner.holds_op(head), journaled);
+ let repaired: Vec<_> = if journaled {
+ prepares.last().into_iter().cloned().collect()
+ } else {
+ prepares.clone()
+ };
+ for prepare in repaired.into_iter().rev() {
+ owner
+ .on_repair_prepare(prepare.transmute_header(
+ |header, repaired: &mut iggy_binary_protocol::RepairPrepareHeader| {
+ repaired.0 = header;
+ repaired.0.command = Command::RepairPrepare;
+ },
+ ))
+ .await;
+ }
+ prepares
+ }
+
+ /// A sealed header at the same op and on the same parent as `header`, naming a
+ /// different entry.
+ fn forked(header: PrepareHeader) -> PrepareHeader {
+ let mut fork = header;
+ fork.checksum_body ^= 1;
+ fork.checksum = fork.identity_checksum();
+ assert_ne!(fork.checksum, header.checksum);
+ fork
+ }
+
#[compio::test]
async fn given_a_blank_start_view_head_when_repair_arrives_should_require_vouched_history() {
const HEAD: u64 = 1;
@@ -7658,6 +7830,7 @@ mod tests {
commit_to_op: STALE_SHAPE_OP,
fetch_to_op: STALE_SHAPE_OP,
floor: None,
+ requested_from: Some(STALE_SHAPE_OP),
peer,
idle_ticks: 0,
});
@@ -7912,6 +8085,7 @@ mod tests {
floor: None,
peer,
idle_ticks: 0,
+ requested_from: Some(commit_min + 1),
});
partition
}
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 7c3b7d4cb..c33d887b1 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -2221,6 +2221,9 @@ fn stamp_metadata_watermark(
}
}
+#[cfg(test)]
+mod stale_prepare_commit_tests;
+
#[cfg(test)]
mod tests {
use super::partition_repair_driver_tests::{
@@ -12569,7 +12572,7 @@ mod partition_repair_driver_tests {
}
/// `(status, view, commit_min, commit_max)` of one replica's partition group.
- fn group_state(
+ pub fn group_state(
sim: &Simulator,
replica: u8,
namespace: IggyNamespace,
diff --git a/core/simulator/src/stale_prepare_commit_tests.rs b/core/simulator/src/stale_prepare_commit_tests.rs
new file mode 100644
index 000000000..3e92fb521
--- /dev/null
+++ b/core/simulator/src/stale_prepare_commit_tests.rs
@@ -0,0 +1,505 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! A replica must never commit a prepare the cluster did not commit.
+//!
+//! The old primary journals an op, is cut off before any peer sees it, and the
+//! other two replicas elect a new view that commits a DIFFERENT prepare at the
+//! same op. Whether the old primary rejoins that view or is later elected over
+//! it, its stale prepare sits below the view's canonical headers, and it must
+//! end up serving the view's entry at that offset, not its own.
+
+use bytes::Bytes;
+use consensus::{PartitionsHandle, Status};
+use futures::FutureExt;
+use iggy_common::PollingStrategy;
+use partitions::{PollingArgs, PollingConsumer};
+use server_common::send_messages::{BatchIntegrity, decode_batch_slice_with};
+use server_common::sharding::IggyNamespace;
+
+use crate::Simulator;
+use crate::client::SimClient;
+use crate::packet::ProcessId;
+use crate::partition_repair_driver_tests::{cluster, group_state};
+
+const REPLICAS: u8 = 3;
+const OLD_PRIMARY: u8 = 0;
+const STEPS_PER_SEND: usize = 12;
+/// Well past `NORMAL_HEARTBEAT_TICKS` (500), so the majority elects.
+const ELECTION_STEPS: usize = 5_000;
+const SETTLE_STEPS: usize = 5_000;
+/// Long enough for a lone survivor's view changes to rotate onto the old primary.
+const REJOIN_STEPS: usize = 40_000;
+
+fn send(sim: &mut Simulator, client: &SimClient, namespace: IggyNamespace, to: u8, payload: &str) {
+ let request = client.send_messages(namespace, &[Bytes::from(payload.to_owned())]);
+ sim.submit_request(client.client_id(), to, request.into_generic());
+ for _ in 0..STEPS_PER_SEND {
+ sim.step();
+ }
+}
+
+fn isolate(sim: &mut Simulator, replica: u8, isolated: bool) {
+ for peer in (0..REPLICAS).filter(|peer| *peer != replica) {
+ sim.network.set_link_filter(
+ ProcessId::Replica(replica),
+ ProcessId::Replica(peer),
+ !isolated,
+ );
+ sim.network.set_link_filter(
+ ProcessId::Replica(peer),
+ ProcessId::Replica(replica),
+ !isolated,
+ );
+ }
+}
+
+/// `(offset, payload)` for every message a consumer can poll from `replica`,
+/// read through the ordinary client poll path.
+fn polled(sim: &mut Simulator, replica: u8, namespace: IggyNamespace) -> Vec<(u64, String)> {
+ let mut out = Vec::new();
+ loop {
+ let next = out.last().map_or(0, |(offset, _)| offset + 1);
+ let args = PollingArgs::new(PollingStrategy::offset(next), 1_000, false);
+ let poll = sim.poll_messages(
+ usize::from(replica),
+ namespace,
+ PollingConsumer::Consumer(1, 0),
+ &args,
+ );
+ let (tx, rx) = shard::channel(1);
+ sim.executor.spawn(async move {
+ let _ = tx.try_send(poll.await);
+ });
+ sim.run_pumps();
+ let fragments = rx
+ .recv()
+ .now_or_never()
+ .expect("poll completes")
+ .expect("reply channel open")
+ .expect("poll accepted");
+ let before = out.len();
+ for fragment in &fragments {
+ let mut bytes = fragment.as_slice();
+ while let Ok(batch) = decode_batch_slice_with(bytes, BatchIntegrity::LayoutOnly) {
+ for message in &batch {
+ out.push((
+ batch.header.base_offset + u64::from(message.header.offset_delta),
+ String::from_utf8_lossy(message.payload).into_owned(),
+ ));
+ }
+ let Some(rest) = bytes.get(batch.header.total_size()..) else {
+ break;
+ };
+ bytes = rest;
+ }
+ }
+ if out.len() == before {
+ return out;
+ }
+ }
+}
+
+#[test]
+fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_rejoins_a_view_that_committed_another_prepare_at_that_op_should_not_commit_its_own()
+ {
+ let (mut sim, client) = cluster(0x5EED_57A1);
+ let namespace = IggyNamespace::new(1, 1, 0);
+ sim.init_partition(namespace);
+ sim.register_client_with_primary(&client);
+
+ for index in 0..3 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ OLD_PRIMARY,
+ &format!("warmup-{index}"),
+ );
+ }
+ let (_, _, warm_commit, _) = group_state(&sim, OLD_PRIMARY, namespace);
+ assert_eq!(warm_commit, 3, "warmup did not commit on the old primary");
+
+ // Cut the old primary off, then hand it a request: it journals op 4 and
+ // can never replicate it.
+ isolate(&mut sim, OLD_PRIMARY, true);
+ send(&mut sim, &client, namespace, OLD_PRIMARY, "stale");
+
+ // The majority elects a new view without it.
+ let new_view = (0..ELECTION_STEPS)
+ .find_map(|_| {
+ sim.step();
+ let (status, view, _, _) = group_state(&sim, 1, namespace);
+ (status == Status::Normal && view > 0).then_some(view)
+ })
+ .expect("replicas 1 and 2 never elected a new view");
+ let new_primary = primary_of(&sim, 1, namespace, new_view);
+ assert_ne!(new_primary, OLD_PRIMARY);
+
+ // The new view commits different prepares at op 4 and beyond.
+ for index in 0..4 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ new_primary,
+ &format!("fresh-{index}"),
+ );
+ }
+ let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace);
+ assert!(
+ majority_commit >= 5,
+ "the new view committed only through {majority_commit}"
+ );
+
+ isolate(&mut sim, OLD_PRIMARY, false);
+ for _ in 0..SETTLE_STEPS {
+ sim.step();
+ }
+ let (_, _, rejoined_commit, _) = group_state(&sim, OLD_PRIMARY, namespace);
+ assert!(
+ rejoined_commit >= majority_commit,
+ "the old primary never caught up: committed {rejoined_commit} of {majority_commit}"
+ );
+
+ let reference = polled(&mut sim, new_primary, namespace);
+ let rejoined = polled(&mut sim, OLD_PRIMARY, namespace);
+ assert!(
+ !reference.iter().any(|(_, payload)| payload == "stale"),
+ "the new view committed the isolated primary's request: {reference:?}"
+ );
+ assert_eq!(
+ rejoined, reference,
+ "the rejoined replica serves a committed log that differs from the view's"
+ );
+}
+
+/// The primary-elect twin: the old primary never adopts the majority's view. It is
+/// elected later, with its stale op under the merged log's commit point, where the
+/// merged headers do not reach.
+#[test]
+fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a_log_that_committed_another_prepare_should_not_commit_its_own()
+ {
+ let (mut sim, client) = cluster(0x5EED_57A1);
+ let namespace = IggyNamespace::new(1, 1, 0);
+ sim.init_partition(namespace);
+ sim.register_client_with_primary(&client);
+ for index in 0..3 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ OLD_PRIMARY,
+ &format!("warmup-{index}"),
+ );
+ }
+ isolate(&mut sim, OLD_PRIMARY, true);
+ send(&mut sim, &client, namespace, OLD_PRIMARY, "stale");
+ let new_view = (0..ELECTION_STEPS)
+ .find_map(|_| {
+ sim.step();
+ let (status, view, _, _) = group_state(&sim, 1, namespace);
+ (status == Status::Normal && view > 0).then_some(view)
+ })
+ .expect("replicas 1 and 2 never elected a new view");
+ let new_primary = primary_of(&sim, 1, namespace, new_view);
+ for index in 0..4 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ new_primary,
+ &format!("fresh-{index}"),
+ );
+ }
+ let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace);
+ assert!(
+ majority_commit >= 5,
+ "the new view committed only through {majority_commit}"
+ );
+
+ // Cut the new primary off and keep the old one apart until the survivor's view
+ // changes reach a view the old primary leads; then let those two meet.
+ let survivor = (0..REPLICAS)
+ .find(|replica| *replica != OLD_PRIMARY && *replica != new_primary)
+ .expect("three replicas");
+ isolate(&mut sim, new_primary, true);
+ (0..REJOIN_STEPS)
+ .find(|_| {
+ sim.step();
+ let (_, view, _, _) = group_state(&sim, survivor, namespace);
+ primary_of(&sim, survivor, namespace, view) == OLD_PRIMARY
+ })
+ .expect("the survivor never reached a view the old primary leads");
+ connect(&mut sim, OLD_PRIMARY, survivor);
+ let led_view = (0..SETTLE_STEPS)
+ .find_map(|_| {
+ sim.step();
+ let (status, view, _, _) = group_state(&sim, OLD_PRIMARY, namespace);
+ (status == Status::Normal
+ && view > new_view
+ && primary_of(&sim, OLD_PRIMARY, namespace, view) == OLD_PRIMARY)
+ .then_some(view)
+ })
+ .expect("the old primary never led a view: the primary-elect path did not run");
+
+ isolate(&mut sim, new_primary, false);
+ for _ in 0..SETTLE_STEPS {
+ sim.step();
+ }
+ let reference = polled(&mut sim, survivor, namespace);
+ let elected = polled(&mut sim, OLD_PRIMARY, namespace);
+ assert!(
+ !reference.iter().any(|(_, payload)| payload == "stale"),
+ "view {led_view} committed the isolated primary's request: {reference:?}"
+ );
+ assert_eq!(
+ elected, reference,
+ "the elected old primary serves a committed log that differs from the survivor's"
+ );
+}
+
+/// A rejoining replica fences everything above its commit point, and while fenced
+/// its commit point cannot move. Repair must still walk a window wider than one
+/// served chunk instead of re-requesting the first chunk forever.
+#[test]
+fn given_a_fenced_window_wider_than_a_repair_chunk_when_the_old_primary_rejoins_should_catch_up() {
+ const CHUNK: u64 = 2;
+ let (mut sim, client) = cluster(0x5EED_57A1);
+ let namespace = IggyNamespace::new(1, 1, 0);
+ sim.init_partition(namespace);
+ sim.register_client_with_primary(&client);
+ for replica in &sim.replicas {
+ for shard in &replica.shards {
+ shard.set_repair_chunk_max(CHUNK);
+ }
+ }
+ for index in 0..3 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ OLD_PRIMARY,
+ &format!("warmup-{index}"),
+ );
+ }
+ isolate(&mut sim, OLD_PRIMARY, true);
+ send(&mut sim, &client, namespace, OLD_PRIMARY, "stale");
+ let new_view = (0..ELECTION_STEPS)
+ .find_map(|_| {
+ sim.step();
+ let (status, view, _, _) = group_state(&sim, 1, namespace);
+ (status == Status::Normal && view > 0).then_some(view)
+ })
+ .expect("replicas 1 and 2 never elected a new view");
+ let new_primary = primary_of(&sim, 1, namespace, new_view);
+ for index in 0..12 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ new_primary,
+ &format!("fresh-{index}"),
+ );
+ }
+ let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace);
+ assert!(
+ majority_commit > 3 + 4 * CHUNK,
+ "the new view committed only through {majority_commit}"
+ );
+
+ isolate(&mut sim, OLD_PRIMARY, false);
+ for _ in 0..SETTLE_STEPS {
+ sim.step();
+ }
+ let (_, _, rejoined_commit, _) = group_state(&sim, OLD_PRIMARY, namespace);
+ assert!(
+ rejoined_commit >= majority_commit,
+ "the old primary never caught up: committed {rejoined_commit} of {majority_commit}"
+ );
+ let reference = polled(&mut sim, new_primary, namespace);
+ let rejoined = polled(&mut sim, OLD_PRIMARY, namespace);
+ assert_eq!(
+ rejoined, reference,
+ "the rejoined replica serves a committed log that differs from the view's"
+ );
+}
+
+/// A proof that breaks cuts the stale suffix from the broken op, and the cut
+/// window must be re-requested at once instead of waiting out the repair retry
+/// timer. Two re-request sites exist: the `RepairDone` of a request already in
+/// flight, and the stall sweep when nothing is in flight. Seed `0x5EED_57A2`
+/// takes the first, seed `0x2` the second.
+#[test]
+fn given_an_old_primary_holding_a_stale_prepare_when_the_proof_cuts_it_should_refetch_the_window_at_once()
+ {
+ for seed in [0x5EED_57A2, 0x2] {
+ let steps = cut_to_catch_up(seed).unwrap_or_else(|| {
+ panic!("seed {seed:#x} left the old primary no stale prepare for the proof to cut")
+ });
+ assert!(
+ steps < usize::try_from(partitions::REPAIR_RETRY_TICKS).unwrap(),
+ "seed {seed:#x}: catching up took {steps} steps after the cut, so the refetch \
+ waited out the repair retry timer"
+ );
+ }
+}
+
+/// Runs the stale-prepare rejoin under `seed` and returns how many steps the
+/// old primary took to catch up after its proof cut the held run, or `None`
+/// when no cut happened (the isolated primary journaled nothing stale).
+fn cut_to_catch_up(seed: u64) -> Option<usize> {
+ let (mut sim, client) = cluster(seed);
+ let namespace = IggyNamespace::new(1, 1, 0);
+ sim.init_partition(namespace);
+ sim.register_client_with_primary(&client);
+ for index in 0..3 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ OLD_PRIMARY,
+ &format!("warmup-{index}"),
+ );
+ }
+ isolate(&mut sim, OLD_PRIMARY, true);
+ for index in 0..3 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ OLD_PRIMARY,
+ &format!("stale-{index}"),
+ );
+ }
+ let new_view = (0..ELECTION_STEPS)
+ .find_map(|_| {
+ sim.step();
+ let (status, view, _, _) = group_state(&sim, 1, namespace);
+ (status == Status::Normal && view > 0).then_some(view)
+ })
+ .expect("replicas 1 and 2 never elected a new view");
+ let new_primary = primary_of(&sim, 1, namespace, new_view);
+ for index in 0..6 {
+ send(
+ &mut sim,
+ &client,
+ namespace,
+ new_primary,
+ &format!("fresh-{index}"),
+ );
+ }
+ let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace);
+
+ isolate(&mut sim, OLD_PRIMARY, false);
+ // The adoption, not the heal, starts the clock: how long the old primary
+ // takes to learn of the new view is the view probe's business.
+ (0..SETTLE_STEPS)
+ .find(|_| {
+ sim.step();
+ group_state(&sim, OLD_PRIMARY, namespace).1 >= new_view
+ })
+ .expect("the old primary never adopted the new view");
+ // The held run only shrinks when a proof breaks and cuts it. Within the
+ // step of the cut a re-request may already have gone out, so the request
+ // is either still pending (`None`) or starts at or below the cut.
+ let mut held = held_through(&sim, OLD_PRIMARY, namespace);
+ let mut cut_at = None;
+ let mut caught_up = None;
+ for step in 0..SETTLE_STEPS {
+ if group_state(&sim, OLD_PRIMARY, namespace).2 >= majority_commit {
+ caught_up = Some(step);
+ break;
+ }
+ let now_held = held_through(&sim, OLD_PRIMARY, namespace);
+ if now_held < held && cut_at.is_none() {
+ cut_at = Some(step);
+ let requested =
+ repair_session(&sim, OLD_PRIMARY, namespace).map(|session| session.requested_from);
+ assert!(
+ requested.is_some_and(|from| from.is_none_or(|from| from <= now_held + 1)),
+ "seed {seed:#x}: the cut left held entries through {now_held} but the \
+ repair request stands at {requested:?}"
+ );
+ }
+ held = now_held;
+ sim.step();
+ }
+ let caught_up = caught_up.unwrap_or_else(|| {
+ panic!("seed {seed:#x}: the old primary never caught up to {majority_commit}")
+ });
+ for _ in 0..SETTLE_STEPS {
+ sim.step();
+ }
+ let reference = polled(&mut sim, new_primary, namespace);
+ let rejoined = polled(&mut sim, OLD_PRIMARY, namespace);
+ assert!(
+ !reference
+ .iter()
+ .any(|(_, payload)| payload.starts_with("stale")),
+ "seed {seed:#x}: the new view committed the isolated primary's requests: {reference:?}"
+ );
+ assert_eq!(
+ rejoined, reference,
+ "seed {seed:#x}: the rejoined replica serves a committed log that differs from the view's"
+ );
+ cut_at.map(|cut_at| caught_up - cut_at)
+}
+
+/// Highest op through which `replica` holds every entry above its commit point.
+fn held_through(sim: &Simulator, replica: u8, namespace: IggyNamespace) -> u64 {
+ sim.replicas[usize::from(replica)]
+ .partition_shard(namespace)
+ .plane
+ .partitions()
+ .get_by_ns(&namespace)
+ .expect("replica hosts the group")
+ .held_through()
+}
+
+/// `replica`'s repair session for the group, if one is armed.
+fn repair_session(
+ sim: &Simulator,
+ replica: u8,
+ namespace: IggyNamespace,
+) -> Option<partitions::RepairSession> {
+ sim.replicas[usize::from(replica)]
+ .partition_shard(namespace)
+ .plane
+ .partitions()
+ .get_by_ns(&namespace)
+ .expect("replica hosts the group")
+ .repair
+}
+
+fn primary_of(sim: &Simulator, replica: u8, namespace: IggyNamespace, view: u32) -> u8 {
+ sim.replicas[usize::from(replica)]
+ .partition_shard(namespace)
+ .plane
+ .partitions()
+ .get_by_ns(&namespace)
+ .expect("replica hosts the group")
+ .consensus()
+ .primary_index(view)
+}
+
+fn connect(sim: &mut Simulator, left: u8, right: u8) {
+ sim.network
+ .set_link_filter(ProcessId::Replica(left), ProcessId::Replica(right), true);
+ sim.network
+ .set_link_filter(ProcessId::Replica(right), ProcessId::Replica(left), true);
+}