You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/07/07 14:06:30 UTC
(iggy) 01/03: watermark
This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch fix_reconnect_panic
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit c103effc1409642bcde2b32f17f793494e9f9134
Author: Grzegorz Koszyk <g....@gmail.com>
AuthorDate: Tue Jul 7 08:43:42 2026 +0200
watermark
---
core/consensus/src/impls.rs | 53 +++++++++++++++---
core/integration/tests/server/purge_delete.rs | 21 ++------
core/metadata/src/impls/recovery.rs | 75 ++++++++++++++++++++++++--
core/partitions/src/iggy_partition.rs | 31 ++++++++++-
core/sdk/src/quic/quic_client.rs | 7 ---
core/sdk/src/session.rs | 12 +++++
core/sdk/src/tcp/tcp_client.rs | 7 ---
core/server-ng/src/bootstrap.rs | 78 +++++++++++++++++----------
core/server-ng/src/partition_helpers.rs | 35 ++++++------
core/server-ng/src/server_error.rs | 12 -----
core/shard/src/lib.rs | 78 ++++++++++++++++++++++++---
core/shard/src/router.rs | 5 ++
12 files changed, 310 insertions(+), 104 deletions(-)
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index 94be74bf1..d17a48c88 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -546,6 +546,19 @@ pub enum Status {
/// Actions to be taken by the caller after processing a VSR event.
#[derive(Debug, Clone)]
+pub enum CommitOutcome {
+ /// Heartbeat ignored (stale view, foreign sender, wrong status, or replay).
+ Ignored,
+ /// Heartbeat accepted; `commit_max` unchanged.
+ Accepted,
+ /// Heartbeat accepted and `commit_max` advanced; run `commit_journal`.
+ Advanced,
+ /// A replica is still heartbeating an older view in which it was
+ /// primary; this replica is the current view's primary and should
+ /// broadcast `StartView` so the stale replica adopts the view.
+ RespondStartView,
+}
+
pub enum VsrAction {
/// Send `StartViewChange` to all replicas.
SendStartViewChange { view: u32, namespace: u64 },
@@ -1721,6 +1734,14 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
}
// Accept the StartView and transition to normal
+ tracing::info!(
+ replica = self.replica,
+ old_view = self.view.get(),
+ new_view = msg_view,
+ op = msg_op,
+ commit = msg_commit,
+ "adopting view from StartView"
+ );
self.view.set(msg_view);
self.log_view.set(msg_view);
self.status.set(Status::Normal);
@@ -1791,29 +1812,45 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// # Panics
/// If `header.namespace` does not match this replica's namespace.
- pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> bool {
+ pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> CommitOutcome {
assert_eq!(
header.namespace, self.namespace,
"Commit routed to wrong group"
);
if self.is_primary() {
- return false;
+ // A heartbeat from the primary of an OLDER view means that
+ // replica missed our view change entirely -- typically it
+ // restarted while the view advanced and recovered the stale
+ // view from its journal (there is no durable view watermark),
+ // so the SVC/DVC/SV exchange never reached it. Left alone it
+ // wedges: it drops our newer-view traffic as foreign and we
+ // drop its stale prepares, while its live heartbeats keep its
+ // backups from electing anyone. Re-announcing the current view
+ // lets its `handle_start_view` adopt the view and cancel its
+ // stale pipeline.
+ if self.status.get() == Status::Normal
+ && header.view < self.view.get()
+ && header.replica == self.primary_index(header.view)
+ {
+ return CommitOutcome::RespondStartView;
+ }
+ return CommitOutcome::Ignored;
}
if self.status.get() != Status::Normal {
- return false;
+ return CommitOutcome::Ignored;
}
if header.view != self.view.get() {
- return false;
+ return CommitOutcome::Ignored;
}
// TODO: Once connection-level peer verification is added promote
// this to an assert, the network layer would guarantee the sender
// matches header.replica.
if header.replica != self.primary_index(header.view) {
- return false;
+ return CommitOutcome::Ignored;
}
// Only accept heartbeats with a strictly newer timestamp to prevent
@@ -1827,7 +1864,11 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
let old_commit_max = self.commit_max.get();
self.advance_commit_max(header.commit);
- self.commit_max.get() > old_commit_max
+ if self.commit_max.get() > old_commit_max {
+ CommitOutcome::Advanced
+ } else {
+ CommitOutcome::Accepted
+ }
}
/// Complete view change as the new primary after collecting DVC quorum.
diff --git a/core/integration/tests/server/purge_delete.rs b/core/integration/tests/server/purge_delete.rs
index 92dfe3539..dc226470f 100644
--- a/core/integration/tests/server/purge_delete.rs
+++ b/core/integration/tests/server/purge_delete.rs
@@ -19,18 +19,14 @@ use crate::server::scenarios::purge_delete_scenario;
use integration::iggy_harness;
use test_case::test_matrix;
-// Legacy-only: asserts the exact on-disk layout ([0, 7, 14, 21], 7 messages per
-// 5KiB segment) and per-file byte sizes throughout. server-ng's per-message
-// framing yields a different layout, so the scenario's offset anchors do not
-// hold under vsr. Port to a framing-agnostic form to re-enable.
-#[cfg(not(feature = "vsr"))]
#[iggy_harness(server(
segment.size = "5KiB",
segment.cache_indexes = ["all", "none", "open_segment"],
partition.messages_required_to_save = "1",
partition.enforce_fsync = "true",
))]
-#[test_matrix([restart_off(), restart_on()])]
+#[cfg_attr(not(feature = "vsr"), test_matrix([restart_off(), restart_on()]))]
+#[cfg_attr(feature = "vsr", test_matrix([restart_off()]))]
async fn should_delete_segments_and_validate_filesystem(
harness: &mut TestHarness,
restart_server: bool,
@@ -53,13 +49,6 @@ async fn should_delete_segments_without_consumers(harness: &mut TestHarness, res
purge_delete_scenario::run_no_consumers(harness, restart_server).await;
}
-// vsr-gated: asserts a segment is freed *exactly* when the polled offset
-// reaches its end — a synchronous-commit assumption. Under vsr the
-// consumer-group auto-commit lags the poll, so the per-message timing does not
-// hold (the barrier itself is correct: the server retains un-consumed
-// segments, verified by `min_committed_offset` + its runtime diag). Legacy-only
-// until redesigned to sync on the committed offset for async commit.
-#[cfg(not(feature = "vsr"))]
#[iggy_harness(server(
segment.size = "5KiB",
segment.cache_indexes = ["all", "none", "open_segment"],
@@ -73,16 +62,14 @@ async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarnes
purge_delete_scenario::run_consumer_group_barrier(&client, &data_path).await;
}
-// Legacy-only: pins the [0, 7, 14, 21] layout across a multi-consumer barrier.
-// server-ng framing differs; port to framing-agnostic to re-enable.
-#[cfg(not(feature = "vsr"))]
#[iggy_harness(server(
segment.size = "5KiB",
segment.cache_indexes = ["all", "none", "open_segment"],
partition.messages_required_to_save = "1",
partition.enforce_fsync = "true",
))]
-#[test_matrix([restart_off(), restart_on()])]
+#[cfg_attr(not(feature = "vsr"), test_matrix([restart_off(), restart_on()]))]
+#[cfg_attr(feature = "vsr", test_matrix([restart_off()]))]
async fn should_block_deletion_until_all_consumers_pass_segment(
harness: &mut TestHarness,
restart_server: bool,
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs
index 4268c1633..91325aaa4 100644
--- a/core/metadata/src/impls/recovery.rs
+++ b/core/metadata/src/impls/recovery.rs
@@ -88,7 +88,16 @@ pub struct RecoveredMetadata<M> {
pub mux_stm: M,
/// `None` means no snapshot existed and no journal entries were replayed.
/// `Some(op)` is the highest op applied, either from the snapshot or journal replay.
+ ///
+ /// Only the committed prefix is applied: an entry's `commit` header field
+ /// carries the primary's commit watermark when the prepare was sent, and
+ /// journaled does not imply committed.
pub last_applied_op: Option<u64>,
+ /// Highest op present in the journal, `None` when it is empty. Ops in
+ /// `(last_applied_op, last_journaled_op]` are prepared-but-uncommitted:
+ /// they stay journal-only until the recovered primary re-replicates them
+ /// (or a backup sees the commit point advance past them).
+ pub last_journaled_op: Option<u64>,
}
/// Recover metadata state from disk.
@@ -138,12 +147,29 @@ where
// must repair or truncate the WAL before the node can boot again.
let headers_to_replay = journal.iter_headers_from(replay_from);
+ // Committed watermark: the `commit` field of each journaled prepare is
+ // the primary's commit point when it was sent (monotone in op order), so
+ // the last entry carries the highest commit this WAL can prove. Ops above
+ // it were prepared but not provably committed; applying them here would
+ // fabricate commit knowledge (a crashed suffix may have been discarded by
+ // a view change) and would hide the suffix from re-replication.
+ let snapshot_floor = snapshot.as_ref().map_or(0, IggySnapshot::sequence_number);
+ let commit_watermark = headers_to_replay
+ .last()
+ .map_or(snapshot_floor, |header| header.commit.max(snapshot_floor));
+
let mut last_applied_op: Option<u64> = None;
+ let mut last_journaled_op: Option<u64> = None;
for header in &headers_to_replay {
// TODO: Check hash chain integrity against `previous_header`. On a
// same-view break, stop replay here and mark the remaining entries for
// repair via VSR instead of panicking.
+ last_journaled_op = Some(header.op);
+ if header.op > commit_watermark {
+ continue;
+ }
+
let entry = journal.entry_at(header).await?.ok_or_else(|| {
RecoveryError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
@@ -161,6 +187,7 @@ where
snapshot,
mux_stm,
last_applied_op,
+ last_journaled_op,
})
}
@@ -180,6 +207,12 @@ mod tests {
const HEADER_SIZE: usize = size_of::<PrepareHeader>();
fn make_prepare(op: u64, body_size: usize) -> Message<PrepareHeader> {
+ make_prepare_with_commit(op, op.saturating_sub(1), body_size)
+ }
+
+ /// A prepare as a live primary stamps it: `commit` carries the primary's
+ /// commit point when the prepare is sent.
+ fn make_prepare_with_commit(op: u64, commit: u64, body_size: usize) -> Message<PrepareHeader> {
let total_size = HEADER_SIZE + body_size;
let mut buffer = Owned::<4096>::zeroed(total_size);
let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(
@@ -188,6 +221,7 @@ mod tests {
header.size = total_size as u32;
header.command = Command2::Prepare;
header.op = op;
+ header.commit = commit;
header.operation = Operation::CreateStream;
Message::try_from(buffer).unwrap()
}
@@ -241,10 +275,44 @@ mod tests {
let recovered = recover::<TestStm>(dir.path()).await.unwrap();
assert!(recovered.snapshot.is_none());
- assert_eq!(recovered.last_applied_op, Some(3));
+ // Op 3's entry proves commit=2; op 3 itself is journaled but not
+ // provably committed, so it stays journal-only.
+ assert_eq!(recovered.last_applied_op, Some(2));
+ assert_eq!(recovered.last_journaled_op, Some(3));
assert_eq!(recovered.journal.last_op(), Some(3));
}
+ #[compio::test]
+ async fn recover_applies_only_the_committed_prefix() {
+ let dir = tempdir().unwrap();
+ let metadata_dir = dir.path().join("metadata");
+ std::fs::create_dir_all(&metadata_dir).unwrap();
+
+ // Ops 1-5 journaled, but the last entry proves commit only up to 3:
+ // the primary crashed before 4 and 5 reached quorum.
+ {
+ let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0)
+ .await
+ .unwrap();
+ for op in 1..=4u64 {
+ journal
+ .append(make_prepare_with_commit(op, op.saturating_sub(1), 32))
+ .await
+ .unwrap();
+ }
+ journal
+ .append(make_prepare_with_commit(5, 3, 32))
+ .await
+ .unwrap();
+ journal.storage_ref().fsync().await.unwrap();
+ }
+
+ let recovered = recover::<TestStm>(dir.path()).await.unwrap();
+ assert_eq!(recovered.last_applied_op, Some(3));
+ assert_eq!(recovered.last_journaled_op, Some(5));
+ assert_eq!(recovered.journal.last_op(), Some(5));
+ }
+
#[compio::test]
async fn recover_snapshot_plus_journal() {
let dir = tempdir().unwrap();
@@ -269,8 +337,9 @@ mod tests {
}
let recovered = recover::<TestStm>(dir.path()).await.unwrap();
- // Should replay ops 6-10 (snapshot was at 5)
- assert_eq!(recovered.last_applied_op, Some(10));
+ // Replays ops 6-9 (snapshot at 5; op 10's entry proves commit=9).
+ assert_eq!(recovered.last_applied_op, Some(9));
+ assert_eq!(recovered.last_journaled_op, Some(10));
assert_eq!(
recovered
.snapshot
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 7373577d2..c171d1174 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -1726,8 +1726,35 @@ where
self.append_messages(message).await.map(|_| ())
}
- #[allow(clippy::too_many_lines)]
async fn commit_messages(&mut self, config: &PartitionsConfig) -> Result<(), IggyError> {
+ self.commit_messages_inner(config, false).await
+ }
+
+ /// Flush the committed journal prefix to segment storage regardless of
+ /// the `messages_required_to_save` thresholds.
+ ///
+ /// Shutdown-path counterpart of the commit-time persist gate: a graceful
+ /// stop must not lose committed messages that were still resident in the
+ /// in-memory journal (consumer offsets are persisted eagerly, so losing
+ /// the messages would fail recovery with an offset ahead of the data).
+ ///
+ /// # Errors
+ ///
+ /// Returns [`IggyError`] when writing the committed batches or their
+ /// index entries to segment storage fails.
+ pub async fn flush_committed_messages(
+ &mut self,
+ config: &PartitionsConfig,
+ ) -> Result<(), IggyError> {
+ self.commit_messages_inner(config, true).await
+ }
+
+ #[allow(clippy::too_many_lines)]
+ async fn commit_messages_inner(
+ &mut self,
+ config: &PartitionsConfig,
+ force: bool,
+ ) -> Result<(), IggyError> {
let write_lock = self.write_lock.clone();
let _guard = write_lock.lock().await;
@@ -1748,7 +1775,7 @@ where
>= config.size_of_messages_required_to_save.as_bytes_u64();
let should_persist =
is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded;
- if !should_persist {
+ if !force && !should_persist {
return Ok(());
}
diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs
index 8b41ac790..6cc24cd0b 100644
--- a/core/sdk/src/quic/quic_client.rs
+++ b/core/sdk/src/quic/quic_client.rs
@@ -177,13 +177,6 @@ impl BinaryTransport for QuicClient {
#[cfg(feature = "vsr")]
if skip_auto_login {
*self.skip_auto_login_once.lock().await = true;
- // The replayed login/register must mint a fresh Register: the failed
- // attempt already consumed the one-shot register request id and may
- // have half-bound the session.
- *self
- .consensus_session
- .lock()
- .expect("consensus session mutex poisoned") = ConsensusSession::new();
}
let server_address = self.current_server_address.lock().await.to_string();
info!(
diff --git a/core/sdk/src/session.rs b/core/sdk/src/session.rs
index ac56c0b33..33bbaaa51 100644
--- a/core/sdk/src/session.rs
+++ b/core/sdk/src/session.rs
@@ -98,6 +98,18 @@ impl ConsensusSession {
self.session.is_some()
}
+ /// Whether a login must mint a fresh session before registering.
+ ///
+ /// The register request id is one-shot per session, so any re-login
+ /// (reconnect auto-login or a deliberate second login) on a consumed or
+ /// bound session requires recreating the session first: new random
+ /// `client_id`, unambiguous re-register; the server evicts the abandoned
+ /// entry (see the module docs).
+ #[must_use]
+ pub fn needs_fresh_register(&self) -> bool {
+ self.register_consumed || self.is_bound()
+ }
+
/// Bind the session after register commits through consensus.
/// The `session` value is the commit op number from the server's reply.
///
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs
index d69ffd2ae..6200179dd 100644
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@ -179,13 +179,6 @@ impl BinaryTransport for TcpClient {
#[cfg(feature = "vsr")]
if skip_auto_login {
*self.skip_auto_login_once.lock().await = true;
- // The replayed login/register must mint a fresh Register: the
- // failed attempt already consumed the one-shot register request
- // id and may have half-bound the session.
- *self
- .consensus_session
- .lock()
- .expect("consensus session mutex poisoned") = ConsensusSession::new();
}
{
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index ca7710148..d93b93feb 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -34,7 +34,9 @@ use configs::server_ng::ServerNgConfig;
use configs::sharding::{
INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX,
};
-use consensus::{LocalPipeline, MetadataHandle, PartitionsHandle, Sequencer, VsrConsensus};
+use consensus::{
+ LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, VsrConsensus,
+};
// `try_send` / `try_recv` resolve through these traits on `MAsyncTx` /
// `MAsyncRx`; the metadata-handoff loops below depend on the
// non-blocking variants for cancel-safe shutdown polling.
@@ -819,6 +821,7 @@ async fn shard_main(
recovered.journal,
recovered.snapshot,
recovered.last_applied_op,
+ recovered.last_journaled_op,
)),
)
}
@@ -838,12 +841,14 @@ async fn shard_main(
// `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`,
// so peer shards have no caller that reads `journal` or `snapshot`.
let (metadata_consensus, journal_for_metadata, snapshot_for_metadata) =
- if let Some((journal, snapshot, last_applied_op)) = owner_state {
- let restored_op = last_applied_op
- .unwrap_or_else(|| snapshot.as_ref().map_or(0, IggySnapshot::sequence_number));
+ if let Some((journal, snapshot, last_applied_op, last_journaled_op)) = owner_state {
+ let snapshot_floor = snapshot.as_ref().map_or(0, IggySnapshot::sequence_number);
+ let commit_watermark = last_applied_op.unwrap_or(snapshot_floor);
+ let restored_op = last_journaled_op.unwrap_or(snapshot_floor);
let consensus = restore_metadata_consensus(
&journal,
restored_op,
+ commit_watermark,
topology.cluster_id,
topology.self_replica_id,
topology.replica_count,
@@ -1482,6 +1487,7 @@ async fn build_shard_for_thread(
fn restore_metadata_consensus(
journal: &PrepareJournal,
restored_op: u64,
+ commit_watermark: u64,
cluster_id: u128,
self_replica_id: u8,
replica_count: u8,
@@ -1506,36 +1512,50 @@ fn restore_metadata_consensus(
consensus.init();
consensus.sequencer().set_sequence(restored_op);
- // TODO(hubcio): clustered bootstrap does not persist a durable
- // (view, commit_op) watermark, so we collapse commit_min/commit_max to
- // `restored_op` (= last journaled op). VSR consequence: on a view
- // change after partial recovery, a replica that came back with a
- // commit_max below the cluster's true commit_min will accept stale
- // prepares as new and overwrite already-committed log entries
- // (split-brain on the committed prefix).
- //
- // Fix direction: persist (view, commit_op) on the journal-header path
- // (`core/journal/src/prepare_journal.rs` PrepareHeader already carries
- // `view`; extend with `commit_op` or add a sidecar watermark file
- // updated on every commit), seed `restore_commit_state(min, max)`
- // from durable state on recovery, and refuse boot if the gap exceeds
- // a configurable threshold. Tracked under the "durable
- // PartitionJournal + durable (view, commit_op) watermark" milestone
- // named in the multi-shard wiring commit body.
+ // The commit point is restored from the WAL's embedded watermark (each
+ // journaled prepare carries the primary's commit at send time), NOT from
+ // the journal head: journaled does not imply committed, and claiming
+ // commit for the un-quorum'd tail both risks split-brain on a later view
+ // change and starves the tail of re-replication (it would live in no
+ // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined
+ // below when this replica is the recovered view's primary.
//
- // Reproducible in `core/simulator` once it grows a restart-from-disk
- // path: today `SimNetwork::enable_process` only un-disables links
- // without replaying `SimJournal` + `SimSnapshot` through
- // `restore_metadata_consensus`, so the flatten cannot trip. Add a
- // crash-restart-replay primitive in the sim, then write a scenario
- // that commits an op on the primary, crashes the primary mid-
- // replicate-ack, triggers a view change, and asserts the recovered
- // replica refuses to re-accept the committed prepare.
- consensus.restore_commit_state(restored_op, restored_op);
+ // TODO(hubcio): the watermark is a lower bound (the last entry stamps
+ // the commit point as of its send). Persisting an explicit (view,
+ // commit_op) watermark on the commit path would tighten recovery and
+ // allow refusing boot on an excessive gap; a backup that recovered a
+ // LONGER tail than the cluster's primary still needs uncommitted-suffix
+ // truncation when conflicting ops arrive (message repair milestone).
+ consensus.restore_commit_state(commit_watermark, commit_watermark);
if let Some(header) = last_header {
consensus.set_last_prepare_checksum(header.checksum);
}
+ // Re-pipeline the prepared-but-uncommitted suffix so the primary's
+ // retransmit machinery re-replicates it and quorum can (re-)commit it.
+ // A backup's suffix stays journal-only: the primary's traffic either
+ // confirms it (re-forward + re-ack path) or supersedes it.
+ if consensus.is_primary() && commit_watermark < restored_op {
+ info!(
+ commit_watermark,
+ restored_op, "re-pipelining recovered uncommitted metadata suffix"
+ );
+ let mut pipeline = consensus.pipeline().borrow_mut();
+ #[allow(clippy::cast_possible_truncation)]
+ for op in (commit_watermark + 1)..=restored_op {
+ let Some(header) = journal.header(op as usize) else {
+ warn!(
+ op,
+ "recovered journal suffix has a gap; stopping re-pipeline"
+ );
+ break;
+ };
+ let mut entry = PipelineEntry::new(*header);
+ entry.add_ack(self_replica_id);
+ pipeline.push(entry);
+ }
+ }
+
consensus
}
diff --git a/core/server-ng/src/partition_helpers.rs b/core/server-ng/src/partition_helpers.rs
index a6aa71fb1..b544e9b4b 100644
--- a/core/server-ng/src/partition_helpers.rs
+++ b/core/server-ng/src/partition_helpers.rs
@@ -41,7 +41,7 @@ use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
-use tracing::error;
+use tracing::{error, warn};
/// Validate that a namespace fits within the static caps declared in
/// `config.extra.namespace`.
@@ -172,10 +172,9 @@ pub async fn create_partition_file_hierarchy(
///
/// # Errors
///
-/// Returns [`ServerNgError::RecoveredConsumerOffsetOutOfBounds`] when a
-/// stored offset exceeds `current_offset`, or
-/// [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files exist
-/// but fail to decode.
+/// Returns [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files
+/// exist but fail to decode. A stored offset ahead of `current_offset` is
+/// clamped (with a warning), not an error.
pub fn configure_consumer_offsets(
partition: &mut IggyPartition<Rc<IggyMessageBus>>,
config: &ServerNgConfig,
@@ -207,15 +206,20 @@ pub fn configure_consumer_offsets(
for offset in loaded_consumer_offsets {
let recovered_offset = offset.offset.load(Ordering::Relaxed);
if recovered_offset > current_offset {
- return Err(ServerNgError::RecoveredConsumerOffsetOutOfBounds {
- consumer_kind: "consumer",
- consumer_id: offset.consumer_id as usize,
- offset: recovered_offset,
+ // A crash can persist an offset ahead of the flushed data
+ // (offsets are stored eagerly, messages flush later). Clamp to
+ // the recovered head so the consumer resumes instead of being
+ // stuck polling past the log; mirrors the legacy contract.
+ warn!(
+ consumer_id = offset.consumer_id,
+ recovered_offset,
current_offset,
stream_id,
topic_id,
partition_id,
- });
+ "recovered consumer offset ahead of partition data; clamping"
+ );
+ offset.offset.store(current_offset, Ordering::Relaxed);
}
guard.insert(offset.consumer_id as usize, offset);
}
@@ -233,15 +237,16 @@ pub fn configure_consumer_offsets(
for (group_id, offset) in loaded_group_offsets {
let recovered_offset = offset.offset.load(Ordering::Relaxed);
if recovered_offset > current_offset {
- return Err(ServerNgError::RecoveredConsumerOffsetOutOfBounds {
- consumer_kind: "consumer group",
- consumer_id: group_id.0,
- offset: recovered_offset,
+ warn!(
+ consumer_group_id = group_id.0,
+ recovered_offset,
current_offset,
stream_id,
topic_id,
partition_id,
- });
+ "recovered consumer group offset ahead of partition data; clamping"
+ );
+ offset.offset.store(current_offset, Ordering::Relaxed);
}
guard.insert(group_id, offset);
}
diff --git a/core/server-ng/src/server_error.rs b/core/server-ng/src/server_error.rs
index 5dc443a9d..041420812 100644
--- a/core/server-ng/src/server_error.rs
+++ b/core/server-ng/src/server_error.rs
@@ -160,18 +160,6 @@ pub enum ServerNgError {
#[source]
source: Box<iggy_common::IggyError>,
},
- #[error(
- "recovered {consumer_kind} offset {offset} for id {consumer_id} exceeds current_offset {current_offset} in stream {stream_id}, topic {topic_id}, partition {partition_id}"
- )]
- RecoveredConsumerOffsetOutOfBounds {
- consumer_kind: &'static str,
- consumer_id: usize,
- offset: u64,
- current_offset: u64,
- stream_id: usize,
- topic_id: usize,
- partition_id: usize,
- },
#[error(
"recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})"
)]
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 3b6b4e60b..7b2b48bc1 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -27,8 +27,8 @@ pub use config::CoordinatorConfig;
#[cfg(any(test, feature = "simulator"))]
use consensus::LocalPipeline;
use consensus::{
- Consensus, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane, PlaneKind, Sequencer,
- VsrAction, VsrConsensus,
+ CommitOutcome, Consensus, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane,
+ PlaneKind, Sequencer, VsrAction, VsrConsensus,
};
use iggy_binary_protocol::{
Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader,
@@ -1768,8 +1768,12 @@ where
if let Some(ref consensus) = planes.0.consensus
&& consensus.namespace() == header.namespace
{
- if consensus.handle_commit(&header) {
- planes.0.commit_journal().await;
+ match consensus.handle_commit(&header) {
+ CommitOutcome::Advanced => planes.0.commit_journal().await,
+ CommitOutcome::RespondStartView => {
+ respond_start_view::<B, _, MJ>(consensus).await;
+ }
+ CommitOutcome::Accepted | CommitOutcome::Ignored => {}
}
return;
}
@@ -1785,8 +1789,12 @@ where
return;
};
let consensus = partition.consensus();
- if consensus.handle_commit(&header) {
- partition.commit_journal(config).await;
+ match consensus.handle_commit(&header) {
+ CommitOutcome::Advanced => partition.commit_journal(config).await,
+ CommitOutcome::RespondStartView => {
+ respond_start_view::<B, _, MJ>(consensus).await;
+ }
+ CommitOutcome::Accepted | CommitOutcome::Ignored => {}
}
}
@@ -1825,6 +1833,34 @@ where
}
}
+ /// Flush every owned partition's committed journal prefix to segment
+ /// storage. Pump-shutdown counterpart of the commit-time persist gate:
+ /// a graceful stop must not lose committed messages still resident in
+ /// the in-memory journal (mirrors the legacy pump's final flush).
+ #[allow(clippy::future_not_send)]
+ pub async fn flush_partitions(&self)
+ where
+ B: MessageBus,
+ {
+ let partitions = self.plane.partitions();
+ let namespaces: Vec<_> = partitions.namespaces().copied().collect();
+ for namespace in namespaces {
+ let Some(partition) = partitions.get_mut_by_ns(&namespace) else {
+ continue;
+ };
+ if let Err(error) = partition
+ .flush_committed_messages(partitions.config())
+ .await
+ {
+ tracing::warn!(
+ namespace_raw = namespace.inner(),
+ %error,
+ "failed to flush partition journal on shutdown"
+ );
+ }
+ }
+ }
+
#[allow(clippy::future_not_send)]
pub async fn tick_metadata(&self)
where
@@ -1859,6 +1895,36 @@ where
}
}
+/// Broadcast a `StartView` for the current view, answering a replica that
+/// still heartbeats an older view (see `CommitOutcome::RespondStartView`).
+#[allow(clippy::future_not_send)]
+async fn respond_start_view<B, P, J>(consensus: &VsrConsensus<B, P>)
+where
+ B: MessageBus,
+ P: Pipeline<Entry = consensus::PipelineEntry>,
+ J: JournalHandle,
+ <J as JournalHandle>::Target: Journal<
+ <J as JournalHandle>::Storage,
+ Entry = Message<PrepareHeader>,
+ Header = PrepareHeader,
+ >,
+{
+ tracing::info!(
+ view = consensus.view(),
+ op = consensus.sequencer().current_sequence(),
+ commit = consensus.commit_max(),
+ namespace = consensus.namespace(),
+ "answering stale-view heartbeat with StartView"
+ );
+ let action = VsrAction::SendStartView {
+ view: consensus.view(),
+ op: consensus.sequencer().current_sequence(),
+ commit: consensus.commit_max(),
+ namespace: consensus.namespace(),
+ };
+ dispatch_vsr_actions::<B, P, J>(consensus, None, &[action]).await;
+}
+
/// Dispatch a list of `VsrAction`s by constructing the appropriate
/// protocol messages and sending them via the consensus message bus.
#[allow(
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index c71b2464d..90ea1a280 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -329,6 +329,11 @@ where
self.apply_reconcile_ops();
}
}
+
+ // Final flush: committed messages still resident in the in-memory
+ // journal must reach segment storage before the process exits, or a
+ // graceful restart recovers consumer offsets ahead of the data.
+ self.flush_partitions().await;
}
/// Sanity check at pump entry: every Consensus frame routed through