You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/08/24 19:33:41 UTC
(iggy) branch master updated: fix(shard): tell an unknown consensus operation from a corrupt header (#3931)
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 2bd9d0bce fix(shard): tell an unknown consensus operation from a corrupt header (#3931)
2bd9d0bce is described below
commit 2bd9d0bce11fc401dc2bec38c126bf9f0ffe8b48
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Mon Aug 24 21:33:37 2026 +0200
fix(shard): tell an unknown consensus operation from a corrupt header (#3931)
---
core/binary_protocol/src/consensus/error.rs | 11 ++
core/binary_protocol/src/consensus/header.rs | 70 ++++++++
core/binary_protocol/src/consensus/mod.rs | 2 +-
core/binary_protocol/src/consensus/operation.rs | 11 ++
core/binary_protocol/src/lib.rs | 3 +-
core/server/src/partition_reconciler.rs | 124 ++++++++++++++
core/server_common/src/consensus_message.rs | 219 +++++++++++++++++++++++-
core/shard/src/metrics.rs | 14 +-
core/shard/src/router.rs | 98 +++++++----
9 files changed, 506 insertions(+), 46 deletions(-)
diff --git a/core/binary_protocol/src/consensus/error.rs b/core/binary_protocol/src/consensus/error.rs
index 02fb044e6..707c8eb54 100644
--- a/core/binary_protocol/src/consensus/error.rs
+++ b/core/binary_protocol/src/consensus/error.rs
@@ -79,6 +79,17 @@ pub enum ConsensusError {
#[error("invalid bit pattern in header (enum discriminant out of range)")]
InvalidBitPattern,
+ // Consequence deliberately left out: `RequestHeader` reaches here too, and a
+ // client frame is dropped without a reply rather than stalling a consensus
+ // group. The plane-specific outcome belongs to the call site that logs it.
+ // "Likely", not asserted: replica frames authenticate the byte (frame seal
+ // or prepare identity) before this is raised, but a client `Request` has
+ // neither checksum, so its byte stays unverifiable.
+ #[error(
+ "operation {operation:#04x} is not known to this build; a newer release likely added it"
+ )]
+ UnsupportedOperation { operation: u8 },
+
#[error("client-bound command {0:?} cannot be dispatched on inbound path")]
ClientBoundCommand(Command),
}
diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs
index 2f205df3d..a9d20ed3c 100644
--- a/core/binary_protocol/src/consensus/header.rs
+++ b/core/binary_protocol/src/consensus/header.rs
@@ -102,6 +102,21 @@ pub fn frame_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 {
/// `aligned_vec::AVec<u8, ConstAlign<16>>` for explicit alignment.
pub trait ConsensusHeader: Sized + CheckedBitPattern + NoUninit {
const COMMAND: Command;
+ /// Whether `checksum` carries the prepare identity, which covers the
+ /// operation byte: an unknown operation on such a frame can be
+ /// authenticated from the raw bytes even though the typed view refuses to
+ /// cast. False wherever `checksum` seals the frame or crosses the client
+ /// boundary.
+ const IDENTITY_CHECKSUMMED: bool = false;
+
+ /// Byte offset of this header's `operation` field, `None` when it carries
+ /// none.
+ ///
+ /// The typed decode reads the raw byte here after a failed checked cast,
+ /// so an operation a newer release added is reported as version skew
+ /// rather than corruption. An offset rather than a getter because the cast
+ /// has already failed by then, so no typed view of the header exists.
+ const OPERATION_OFFSET: Option<usize> = None;
/// Whether a frame carrying `command` may be typed as this header.
/// Defaults to an exact match; a header that serves several commands
@@ -477,6 +492,7 @@ fn validate_request_fields(
}
impl ConsensusHeader for RoutedRequestHeader {
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(Self, operation));
const COMMAND: Command = Command::Request;
/// The client-wire [`RequestHeader`] this is promoted from is unsealed, and the
/// promotion copies `checksum` verbatim, so there is nothing here to verify.
@@ -511,6 +527,7 @@ impl ConsensusHeader for RoutedRequestHeader {
}
impl ConsensusHeader for RequestHeader {
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(Self, operation));
const COMMAND: Command = Command::Request;
const FRAME_SEALED: bool = false;
@@ -621,6 +638,7 @@ impl Default for ReplyHeader {
}
impl ConsensusHeader for ReplyHeader {
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(Self, operation));
const COMMAND: Command = Command::Reply;
const FRAME_SEALED: bool = false;
@@ -971,6 +989,8 @@ impl Default for PrepareHeader {
}
impl ConsensusHeader for PrepareHeader {
+ const IDENTITY_CHECKSUMMED: bool = true;
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(Self, operation));
const COMMAND: Command = Command::Prepare;
const FRAME_SEALED: bool = false;
@@ -1037,6 +1057,23 @@ pub fn frame_body(frame: &[u8], size: u32) -> &[u8] {
&frame[HEADER_SIZE..end]
}
+/// Byte-level twin of [`PrepareHeader::identity_checksum`].
+///
+/// For a frame whose operation byte does not cast: an unknown discriminant
+/// blocks the typed view, but the identity covers the operation byte, so
+/// recomputing it over the raw bytes authenticates that byte before anything
+/// is claimed about it. Must agree with the typed method byte for byte; a unit
+/// test pins the equality.
+#[must_use]
+pub fn prepare_identity_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 {
+ let mut covered = *header;
+ let checksum_at = core::mem::offset_of!(PrepareHeader, checksum);
+ covered[checksum_at..checksum_at + size_of::<u128>()].fill(0);
+ let view_at = core::mem::offset_of!(PrepareHeader, view);
+ covered[view_at..view_at + size_of::<u32>()].fill(0);
+ u128::from(twox_hash::XxHash3_64::oneshot(&covered))
+}
+
impl PrepareHeader {
/// Which prepare this is, independent of which view re-sent it.
///
@@ -1069,6 +1106,12 @@ impl PrepareHeader {
pub struct RepairPrepareHeader(pub PrepareHeader);
impl ConsensusHeader for RepairPrepareHeader {
+ // Transparent over `PrepareHeader`, so the operation sits at the same
+ // offset and its checksum field carries the same identity. Without these a
+ // repaired prepare carrying a newer release's operation is classified as a
+ // corrupt header instead of version skew.
+ const IDENTITY_CHECKSUMMED: bool = true;
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(PrepareHeader, operation));
const COMMAND: Command = Command::RepairPrepare;
const FRAME_SEALED: bool = false;
@@ -1178,6 +1221,7 @@ impl Default for PrepareOkHeader {
}
impl ConsensusHeader for PrepareOkHeader {
+ const OPERATION_OFFSET: Option<usize> = Some(core::mem::offset_of!(Self, operation));
const FRAME_SEALED: bool = true;
const COMMAND: Command = Command::PrepareOk;
@@ -2693,6 +2737,32 @@ fn validate_forward_frame(
#[cfg(test)]
mod tests {
+
+ // The byte-level twin exists for frames whose operation byte does not
+ // cast; if it ever disagrees with the typed method, a genuine
+ // newer-release prepare would be reported as corruption.
+ #[test]
+ fn byte_level_identity_checksum_matches_the_typed_method() {
+ let mut header = PrepareHeader {
+ command: Command::Prepare,
+ view: 42,
+ op: 7,
+ client: 0xCAFE,
+ operation: Operation::CreateStream,
+ size: u32::try_from(HEADER_SIZE).expect("header size fits in u32"),
+ ..Default::default()
+ };
+ header.checksum = header.identity_checksum();
+
+ let bytes: &[u8; HEADER_SIZE] = bytemuck::bytes_of(&header)
+ .try_into()
+ .expect("a consensus header is HEADER_SIZE bytes");
+ assert_eq!(
+ prepare_identity_checksum_bytes(bytes),
+ header.identity_checksum()
+ );
+ }
+ use super::prepare_identity_checksum_bytes;
use super::{
Command, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader,
EvictionReason, ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader,
diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs
index d33f842eb..7d86503d2 100644
--- a/core/binary_protocol/src/consensus/mod.rs
+++ b/core/binary_protocol/src/consensus/mod.rs
@@ -53,7 +53,7 @@ pub use header::{
RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader,
RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader,
StartViewHeader, StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes,
- read_size_field,
+ prepare_identity_checksum_bytes, read_size_field,
};
pub use operation::Operation;
pub use reply_result::{RESULT_COUNT_LEN, RESULT_ENTRY_LEN, result_code, result_section_len};
diff --git a/core/binary_protocol/src/consensus/operation.rs b/core/binary_protocol/src/consensus/operation.rs
index 691bd9f1b..95de1d4ee 100644
--- a/core/binary_protocol/src/consensus/operation.rs
+++ b/core/binary_protocol/src/consensus/operation.rs
@@ -98,6 +98,17 @@ pub enum Operation {
}
impl Operation {
+ /// Whether `code` is a discriminant this build defines.
+ ///
+ /// The typed decode needs this to tell an operation a newer release added
+ /// from a corrupted header byte: bytemuck's checked cast rejects both with
+ /// one undifferentiated error, and only the former is fixable by upgrading
+ /// this node.
+ #[must_use]
+ pub fn is_known_code(code: u8) -> bool {
+ bytemuck::checked::try_cast::<u8, Self>(code).is_ok()
+ }
+
pub const INTERNAL_START: u8 = Self::CreateTopicWithAssignments as u8;
pub const METADATA_START: u8 = Self::CreateStream as u8;
pub const PARTITION_START: u8 = Self::SendMessages as u8;
diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs
index 7ec2ddd93..bdbcd2c2f 100644
--- a/core/binary_protocol/src/lib.rs
+++ b/core/binary_protocol/src/lib.rs
@@ -78,7 +78,8 @@ pub use consensus::{
ReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader,
RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET,
StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader,
- frame_body, frame_checksum_bytes, read_size_field, result_code, result_section_len,
+ frame_body, frame_checksum_bytes, prepare_identity_checksum_bytes, read_size_field,
+ result_code, result_section_len,
};
pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command};
pub use error::WireError;
diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs
index 6b7b94e2a..699e9c94d 100644
--- a/core/server/src/partition_reconciler.rs
+++ b/core/server/src/partition_reconciler.rs
@@ -2395,6 +2395,130 @@ mod tests {
);
}
+ /// The mixed-version upgrade hole from IGGY-250, at the router seam this
+ /// time: a wire-valid consensus frame carrying an operation only a newer
+ /// release defines must leave an accounted, operator-visible trace instead
+ /// of a bare warn log, because the frame's group stops making progress
+ /// until this node is upgraded.
+ #[compio::test]
+ async fn given_an_unknown_operation_when_dispatched_should_account_an_upgrade_fence_drop() {
+ // Far past every discriminant this build defines.
+ const OPERATION_FROM_A_NEWER_RELEASE: u8 = 0xEE;
+
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let shard = build_test_shard(0, &config, TestMux::default());
+
+ let mut owned = server_common::iobuf::Owned::<{ server_common::MESSAGE_ALIGN }>::zeroed(
+ iggy_binary_protocol::HEADER_SIZE,
+ );
+ {
+ let frame = owned.as_mut_slice();
+ let size_offset = std::mem::offset_of!(PrepareHeader, size);
+ let frame_size =
+ u32::try_from(iggy_binary_protocol::HEADER_SIZE).expect("header size fits in u32");
+ frame[size_offset..size_offset + 4].copy_from_slice(&frame_size.to_le_bytes());
+ frame[std::mem::offset_of!(PrepareHeader, command)] = Command::Prepare as u8;
+ frame[std::mem::offset_of!(PrepareHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE;
+ // A prepare's `checksum` carries the identity checksum, computed
+ // over the operation byte among others; stamping it the way a real
+ // sender does is what lets the classifier trust that byte.
+ let header: &[u8; iggy_binary_protocol::HEADER_SIZE] = frame
+ [..iggy_binary_protocol::HEADER_SIZE]
+ .try_into()
+ .expect("frame spans a full header");
+ let identity = iggy_binary_protocol::prepare_identity_checksum_bytes(header);
+ frame[..size_of::<u128>()].copy_from_slice(&identity.to_le_bytes());
+ }
+ let message = Message::<iggy_binary_protocol::GenericHeader>::try_from(owned)
+ .expect("an identity-stamped Prepare frame is wire-valid in the generic view");
+
+ let before = shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION,
+ );
+ shard.dispatch(message);
+ assert_eq!(
+ shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION,
+ ),
+ before + 1,
+ "an operation from a newer release must be accounted under its own reason, not \
+ folded into the generic unparsable drop"
+ );
+ assert_eq!(
+ shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNPARSABLE,
+ ),
+ 0,
+ "version skew must not read as header corruption"
+ );
+ }
+
+ /// The complement of the upgrade-fence accounting: an operation byte that
+ /// landed in undefined space by CORRUPTION (the prepare identity no longer
+ /// matches) must count as `unparsable`, never as version skew -- the two
+ /// counters send an operator to opposite remedies.
+ #[compio::test]
+ async fn given_a_corrupt_operation_byte_when_dispatched_should_account_an_unparsable_drop() {
+ const OPERATION_FROM_A_NEWER_RELEASE: u8 = 0xEE;
+
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let shard = build_test_shard(0, &config, TestMux::default());
+
+ let mut owned = server_common::iobuf::Owned::<{ server_common::MESSAGE_ALIGN }>::zeroed(
+ iggy_binary_protocol::HEADER_SIZE,
+ );
+ {
+ let frame = owned.as_mut_slice();
+ let size_offset = std::mem::offset_of!(PrepareHeader, size);
+ let frame_size =
+ u32::try_from(iggy_binary_protocol::HEADER_SIZE).expect("header size fits in u32");
+ frame[size_offset..size_offset + 4].copy_from_slice(&frame_size.to_le_bytes());
+ frame[std::mem::offset_of!(PrepareHeader, command)] = Command::Prepare as u8;
+ // Identity stamped over operation 0, then the byte flipped in
+ // flight: the checksum mismatch is what proves corruption.
+ let header: &[u8; iggy_binary_protocol::HEADER_SIZE] = frame
+ [..iggy_binary_protocol::HEADER_SIZE]
+ .try_into()
+ .expect("frame spans a full header");
+ let identity = iggy_binary_protocol::prepare_identity_checksum_bytes(header);
+ frame[..size_of::<u128>()].copy_from_slice(&identity.to_le_bytes());
+ frame[std::mem::offset_of!(PrepareHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE;
+ }
+ let message = Message::<iggy_binary_protocol::GenericHeader>::try_from(owned)
+ .expect("the frame is wire-valid in the generic view");
+
+ let unparsable_before = shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNPARSABLE,
+ );
+ let skew_before = shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION,
+ );
+ shard.dispatch(message);
+ assert_eq!(
+ shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNPARSABLE,
+ ),
+ unparsable_before + 1,
+ "a corrupt operation byte is header damage and must be accounted as such"
+ );
+ assert_eq!(
+ shard.metrics().frame_drop_count(
+ shard::metrics::frame_drop_variant::CONSENSUS,
+ shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION,
+ ),
+ skew_before,
+ "corruption must not tell an operator to upgrade the node"
+ );
+ }
+
/// Receive half of the purge gate in `on_repair_range_reply`: while a
/// committed purge has not applied locally, a repair verdict must be
/// deferred wholesale -- installing the peer's floor against pre-purge
diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs
index cd4cd91e1..ec7ff6ba0 100644
--- a/core/server_common/src/consensus_message.rs
+++ b/core/server_common/src/consensus_message.rs
@@ -20,11 +20,12 @@ use crate::sharding::METADATA_GROUP;
use iggy_binary_protocol::{
Command, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader,
ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader,
- ForwardRegisterResultHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader,
- RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader,
- RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader,
- RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader,
- StateTransferTargetHeader,
+ ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, Operation, PrepareHeader,
+ PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader,
+ RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader,
+ RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader,
+ StateChunkHeader, StateTransferTargetHeader, frame_checksum_bytes,
+ prepare_identity_checksum_bytes,
};
use smallvec::SmallVec;
use std::{
@@ -242,7 +243,7 @@ where
let bytes = <B as MessageBacking<T>>::header_storage(&self.backing);
let typed = bytemuck::checked::try_from_bytes::<T>(&bytes[..size_of::<T>()])
- .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ .map_err(|_| classify_failed_cast::<T>(bytes))?;
// Before `validate`: a header that did not survive the link intact cannot
// have any of its fields believed, and `validate` reads them.
typed.verify_frame()?;
@@ -281,7 +282,7 @@ where
let bytes = <B as MessageBacking<T>>::header_storage(&self.backing);
let typed = bytemuck::checked::try_from_bytes::<T>(&bytes[..size_of::<T>()])
- .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ .map_err(|_| classify_failed_cast::<T>(bytes))?;
// Before `validate`: a header that did not survive the link intact cannot
// have any of its fields believed, and `validate` reads them.
typed.verify_frame()?;
@@ -477,7 +478,7 @@ where
}
let header = bytemuck::checked::try_from_bytes::<H>(&bytes[..size_of::<H>()])
- .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ .map_err(|_| classify_failed_cast::<H>(bytes))?;
header.validate()?;
// `size` is the whole-frame length and must at least span the header, or
@@ -524,7 +525,7 @@ where
}
let header = bytemuck::checked::try_from_bytes::<H>(&first.as_slice()[..size_of::<H>()])
- .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ .map_err(|_| classify_failed_cast::<H>(first.as_slice()))?;
header.validate()?;
// See `TryFrom<Owned>`: `size` must at least span the header so a
@@ -774,6 +775,68 @@ impl MessageBag {
}
}
+/// Why `H`'s header bytes failed bytemuck's checked cast.
+///
+/// An operation discriminant this build does not define means the sender runs a
+/// newer release; the frame is wire-valid and the node needs upgrading, which is
+/// a different operator action from the corrupted-header case. bytemuck reports
+/// both as one error, so the operation byte is probed here to separate them.
+fn classify_failed_cast<H>(bytes: &[u8]) -> ConsensusError
+where
+ H: ConsensusHeader,
+{
+ if let Some(offset) = H::OPERATION_OFFSET
+ && let Some(&code) = bytes.get(offset)
+ && !Operation::is_known_code(code)
+ && operation_byte_is_trustworthy::<H>(bytes)
+ {
+ return ConsensusError::UnsupportedOperation { operation: code };
+ }
+ ConsensusError::InvalidBitPattern
+}
+
+/// Whether the operation byte may be read as the sender's release rather than
+/// as damage.
+///
+/// This classification runs before `verify_frame`, so the byte is still
+/// unverified: a flipped bit landing in an undefined discriminant would
+/// otherwise be reported as "upgrade this node", and for a `PrepareOk` --
+/// which echoes an operation this primary minted itself -- corruption is the
+/// likelier cause anyway. Sealed headers verify the frame checksum; the
+/// Prepare family verifies the identity checksum, which covers the operation
+/// byte, recomputed over the raw bytes because the unknown discriminant blocks
+/// the typed view.
+///
+/// A `Request` has neither (its `checksum` crosses the client boundary
+/// unsealed), so its byte is taken at face value and the error text claims
+/// only *likely* skew.
+fn operation_byte_is_trustworthy<H>(bytes: &[u8]) -> bool
+where
+ H: ConsensusHeader,
+{
+ if !H::FRAME_SEALED && !H::IDENTITY_CHECKSUMMED {
+ return true;
+ }
+ let Some(header) = bytes
+ .get(..HEADER_SIZE)
+ .and_then(|slice| <&[u8; HEADER_SIZE]>::try_from(slice).ok())
+ else {
+ return false;
+ };
+ let Some(stored) = header
+ .get(..size_of::<u128>())
+ .and_then(|slice| <[u8; size_of::<u128>()]>::try_from(slice).ok())
+ else {
+ return false;
+ };
+ let expected = if H::FRAME_SEALED {
+ frame_checksum_bytes(header)
+ } else {
+ prepare_identity_checksum_bytes(header)
+ };
+ u128::from_le_bytes(stored) == expected
+}
+
impl<T> TryFrom<Message<T>> for MessageBag
where
T: ConsensusHeader,
@@ -919,6 +982,144 @@ mod tests {
owned
}
+ /// An operation byte from a newer release, on a frame whose seal proves the
+ /// byte arrived as the sender wrote it.
+ fn sealed_prepare_ok_with_operation(operation: u8) -> Owned<MESSAGE_ALIGN> {
+ let mut owned = Owned::<MESSAGE_ALIGN>::zeroed(HEADER_SIZE);
+ {
+ let buf = owned.as_mut_slice();
+ buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(HEADER_SIZE as u32).to_le_bytes());
+ buf[COMMAND_OFF] = Command::PrepareOk as u8;
+ buf[offset_of!(PrepareOkHeader, operation)] = operation;
+ seal_header_bytes(buf);
+ }
+ owned
+ }
+
+ const OPERATION_FROM_A_NEWER_RELEASE: u8 = 0xEE;
+
+ #[test]
+ fn given_a_sealed_frame_when_its_operation_is_unknown_should_report_version_skew() {
+ let owned = sealed_prepare_ok_with_operation(OPERATION_FROM_A_NEWER_RELEASE);
+
+ assert!(matches!(
+ classify_failed_cast::<PrepareOkHeader>(owned.as_slice()),
+ ConsensusError::UnsupportedOperation {
+ operation: OPERATION_FROM_A_NEWER_RELEASE
+ }
+ ));
+ }
+
+ // The classification runs before `verify_frame`, so on a sealed header the
+ // operation byte is unverified. A flipped bit landing in an undefined
+ // discriminant must not be reported as "the sender runs a newer release":
+ // that sends an operator upgrading a node over a corrupt link, and a
+ // `PrepareOk` echoes an operation this primary minted itself, so corruption
+ // is the likelier cause to begin with.
+ #[test]
+ fn given_a_corrupt_sealed_frame_when_its_operation_is_unknown_should_not_claim_version_skew() {
+ let mut owned = sealed_prepare_ok_with_operation(0);
+ // Flip the operation byte AFTER sealing, the way damage in flight does.
+ owned.as_mut_slice()[offset_of!(PrepareOkHeader, operation)] =
+ OPERATION_FROM_A_NEWER_RELEASE;
+
+ assert!(matches!(
+ classify_failed_cast::<PrepareOkHeader>(owned.as_slice()),
+ ConsensusError::InvalidBitPattern
+ ));
+ }
+
+ /// A Prepare frame stamped the way a real sender stamps it: `checksum`
+ /// carries the prepare identity, which covers the operation byte.
+ fn identity_stamped_prepare(command: Command, operation: u8) -> Owned<MESSAGE_ALIGN> {
+ let mut owned = Owned::<MESSAGE_ALIGN>::zeroed(HEADER_SIZE);
+ {
+ let buf = owned.as_mut_slice();
+ buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(HEADER_SIZE as u32).to_le_bytes());
+ buf[COMMAND_OFF] = command as u8;
+ buf[offset_of!(PrepareHeader, operation)] = operation;
+ let header: &[u8; HEADER_SIZE] =
+ buf[..HEADER_SIZE].try_into().expect("frame is a header");
+ let identity = prepare_identity_checksum_bytes(header);
+ buf[..size_of::<u128>()].copy_from_slice(&identity.to_le_bytes());
+ }
+ owned
+ }
+
+ // A prepare's `checksum` carries its identity, which covers the operation
+ // byte, so an intact identity authenticates the unknown byte as the
+ // sender's release rather than damage.
+ #[test]
+ fn given_an_identity_stamped_prepare_when_its_operation_is_unknown_should_report_version_skew()
+ {
+ let owned = identity_stamped_prepare(Command::Prepare, OPERATION_FROM_A_NEWER_RELEASE);
+
+ const { assert!(!PrepareHeader::FRAME_SEALED) };
+ assert!(matches!(
+ classify_failed_cast::<PrepareHeader>(owned.as_slice()),
+ ConsensusError::UnsupportedOperation {
+ operation: OPERATION_FROM_A_NEWER_RELEASE
+ }
+ ));
+ }
+
+ // The identity was computed over a DIFFERENT operation byte, so the flip
+ // happened after stamping: damage, not a release.
+ #[test]
+ fn given_a_corrupt_prepare_when_its_operation_is_unknown_should_not_claim_version_skew() {
+ let mut owned = identity_stamped_prepare(Command::Prepare, 0);
+ owned.as_mut_slice()[offset_of!(PrepareHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE;
+
+ assert!(matches!(
+ classify_failed_cast::<PrepareHeader>(owned.as_slice()),
+ ConsensusError::InvalidBitPattern
+ ));
+ }
+
+ // A client `Request` has neither a frame seal nor a prepare identity, so
+ // its operation byte is taken at face value; the error text claims only
+ // likely skew for exactly this reason.
+ #[test]
+ fn given_a_client_request_when_its_operation_is_unknown_should_report_version_skew() {
+ let mut owned = Owned::<MESSAGE_ALIGN>::zeroed(HEADER_SIZE);
+ {
+ let buf = owned.as_mut_slice();
+ buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(HEADER_SIZE as u32).to_le_bytes());
+ buf[COMMAND_OFF] = Command::Request as u8;
+ buf[offset_of!(RequestHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE;
+ }
+
+ const { assert!(!RequestHeader::FRAME_SEALED && !RequestHeader::IDENTITY_CHECKSUMMED) };
+ assert!(matches!(
+ classify_failed_cast::<RequestHeader>(owned.as_slice()),
+ ConsensusError::UnsupportedOperation {
+ operation: OPERATION_FROM_A_NEWER_RELEASE
+ }
+ ));
+ }
+
+ // Transparent over `PrepareHeader`, so a repaired prepare from a newer
+ // release must reach the same fence instead of reading as corruption.
+ #[test]
+ fn given_a_repaired_prepare_when_its_operation_is_unknown_should_report_version_skew() {
+ assert_eq!(
+ RepairPrepareHeader::OPERATION_OFFSET,
+ PrepareHeader::OPERATION_OFFSET,
+ "the newtype carries the same operation field"
+ );
+ const { assert!(RepairPrepareHeader::IDENTITY_CHECKSUMMED) };
+
+ let owned =
+ identity_stamped_prepare(Command::RepairPrepare, OPERATION_FROM_A_NEWER_RELEASE);
+
+ assert!(matches!(
+ classify_failed_cast::<RepairPrepareHeader>(owned.as_slice()),
+ ConsensusError::UnsupportedOperation {
+ operation: OPERATION_FROM_A_NEWER_RELEASE
+ }
+ ));
+ }
+
#[test]
fn given_a_sealed_do_view_change_when_dispatching_should_accept() {
let generic = Message::<GenericHeader>::try_from(sealed_do_view_change())
diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs
index 99f70a224..cd05dba02 100644
--- a/core/shard/src/metrics.rs
+++ b/core/shard/src/metrics.rs
@@ -120,6 +120,14 @@ pub mod frame_drop_variant {
/// replicated traffic has nobody to answer, so this is the only record the op
/// was destroyed.
pub mod frame_drop_reason {
+ /// Operation discriminant unknown to this build: the sender is newer.
+ ///
+ /// Distinct from `UNPARSABLE` because upgrading this node is the fix, and
+ /// until it is, the frame's consensus group gap-stops here.
+ pub const UNSUPPORTED_OPERATION: &str = "unsupported_operation";
+ /// A consensus frame failed typed decode for any other reason (corrupt
+ /// header, bad size, client-bound command on the inbound path).
+ pub const UNPARSABLE: &str = "unparsable";
pub const FULL: &str = "full";
pub const DISCONNECTED: &str = "disconnected";
pub const UNROUTABLE: &str = "unroutable";
@@ -131,10 +139,10 @@ pub mod frame_drop_reason {
// The tables only index the lazy fast-path cache below; a `{variant, reason}`
// pair enters the `Family` (and therefore the scrape) the first time a drop
-// site actually produces it, so the unreachable corners of the 7 x 7 cross
+// site actually produces it, so the unreachable corners of the 7 x 9 cross
// product never appear as permanent zero-valued series.
const VARIANT_COUNT: usize = 7;
-const REASON_COUNT: usize = 7;
+const REASON_COUNT: usize = 9;
const VARIANTS: [&str; VARIANT_COUNT] = [
frame_drop_variant::CONSENSUS,
@@ -147,6 +155,8 @@ const VARIANTS: [&str; VARIANT_COUNT] = [
];
const REASONS: [&str; REASON_COUNT] = [
+ frame_drop_reason::UNSUPPORTED_OPERATION,
+ frame_drop_reason::UNPARSABLE,
frame_drop_reason::FULL,
frame_drop_reason::DISCONNECTED,
frame_drop_reason::UNROUTABLE,
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index 4340761a0..43f02c627 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -23,7 +23,7 @@ use crate::{IggyShard, LifecycleFrame, Receiver, RestorableMetadataStm, ShardFra
use consensus::{MetadataHandle, PartitionsHandle};
use crossfire::TrySendError;
use futures::FutureExt;
-use iggy_binary_protocol::{GenericHeader, Operation, PrepareHeader};
+use iggy_binary_protocol::{Command, ConsensusError, GenericHeader, Operation, PrepareHeader};
use journal::superblock::SuperblockStore;
use journal::{Journal, JournalHandle};
use message_bus::{ConnectionInstaller, MessageBus, ReplicaHandshakeDoneFn};
@@ -60,19 +60,48 @@ where
/// made every frame pay `bytemuck::checked::try_from_bytes` plus the
/// header's `validate()` twice.
pub fn dispatch(&self, message: Message<GenericHeader>) {
+ let command = message.header().command;
let bag = match MessageBag::try_from(message) {
Ok(bag) => bag,
+ Err(ConsensusError::UnsupportedOperation { operation }) => {
+ // For a replication frame this is terminal for its consensus
+ // group, not a per-frame hiccup: the op is never journaled,
+ // never acked, and every later prepare dies on the resulting
+ // gap while quorum hides the outage. Repair wraps the same
+ // typed header, so it cannot rescue this node either -- only
+ // upgrading it can. A routed request, by contrast, is dropped
+ // before journaling and stalls nothing; the consequence in the
+ // log follows the frame. Nothing fences the sending peer, so
+ // the log and the counter are the whole signal an operator
+ // gets.
+ self.metrics.record_frame_drop(
+ frame_drop_variant::CONSENSUS,
+ frame_drop_reason::UNSUPPORTED_OPERATION,
+ );
+ let consequence = match command {
+ Command::Prepare | Command::RepairPrepare | Command::PrepareOk => {
+ "this node cannot journal or ack it, so its consensus group stops \
+ making progress until this node is upgraded"
+ }
+ _ => "the frame is dropped before journaling, with no reply to the sender",
+ };
+ tracing::error!(
+ shard = self.id,
+ operation = format_args!("{operation:#04x}"),
+ command = ?command,
+ build_release = %iggy_binary_protocol::ProtocolVersion(
+ iggy_binary_protocol::IGGY_PROTOCOL_VERSION
+ ),
+ "frame carries an operation this build does not know; a newer release \
+ likely added it. {consequence}"
+ );
+ return;
+ }
Err(e) => {
- // TODO(hubcio): this drop is the whole story for a consensus
- // frame carrying an Operation this build does not know: no
- // metric, no peer error, no eviction. An old node in a mixed
- // cluster silently gap-stops the group here (never journals
- // the op, never PrepareOks, every later prepare dies on the
- // gap check) while quorum hides it, and repair wraps the same
- // typed header so it cannot rescue. Rolling upgrades across
- // consensus-op additions need a version fence (release_min /
- // release_max bounds on the replica plane) before this arm is
- // safe to hit.
+ self.metrics.record_frame_drop(
+ frame_drop_variant::CONSENSUS,
+ frame_drop_reason::UNPARSABLE,
+ );
tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame");
return;
}
@@ -798,34 +827,30 @@ enum ReplicaHandshakeOutcome {
#[cfg(test)]
mod tests {
use iggy_binary_protocol::{
- Command, ConsensusError, GenericHeader, HEADER_SIZE, PrepareHeader, frame_checksum_bytes,
+ Command, ConsensusError, GenericHeader, HEADER_SIZE, PrepareHeader,
+ prepare_identity_checksum_bytes,
};
use server_common::iobuf::Owned;
use server_common::{MESSAGE_ALIGN, Message, MessageBag};
use std::mem::offset_of;
- /// RED SPEC, expected to FAIL: pins the mixed-cluster upgrade hole in
- /// `dispatch`'s decode seam.
- ///
/// An `Operation` discriminant this build does not know, arriving on an
/// otherwise wire-valid consensus frame (correct command, size, checksum:
- /// exactly what a newer release sends after an op addition), decodes to
- /// the same undifferentiated `ConsensusError::InvalidBitPattern` as random
- /// memory corruption. `dispatch` answers both identically: a warn log and
- /// a dropped frame. No metric, no peer error, no eviction, no version
- /// fence. An old node in a mixed cluster therefore gap-stops its consensus
- /// group silently (never journals the op, never sends a `PrepareOk`, every
- /// later prepare dies on the gap check) while quorum hides the outage.
+ /// exactly what a newer release sends after an op addition), must decode to
+ /// its own error rather than the `InvalidBitPattern` that random memory
+ /// corruption produces.
///
- /// Passes once the decode surfaces a dedicated unsupported-operation
- /// signal the router can fence and account, instead of collapsing it into
- /// the corruption error.
+ /// The two need different operator actions: version skew is fixed by
+ /// upgrading this node, and until it is, the frame's consensus group makes
+ /// no progress (the op is never journaled, never acked, and every later
+ /// prepare dies on the gap). `dispatch` splits its drop arms on this
+ /// distinction; the accounting half is pinned in the server crate by
+ /// `given_an_unknown_operation_when_dispatched_should_account_an_upgrade_fence_drop`.
#[test]
- // TODO(hubcio): fix this test
- #[ignore = "unknown operation collapses into InvalidBitPattern; no upgrade fence"]
fn given_an_unknown_operation_when_a_consensus_frame_decodes_should_surface_an_upgrade_fence_signal()
{
- // Far past every defined Operation discriminant (the highest is 165).
+ // Far past every defined Operation discriminant (the highest is 162,
+ // `DeleteConsumerOffset`).
const OPERATION_FROM_A_NEWER_RELEASE: u8 = 0xEE;
let mut owned = Owned::<MESSAGE_ALIGN>::zeroed(HEADER_SIZE);
@@ -838,13 +863,15 @@ mod tests {
let client_offset = offset_of!(PrepareHeader, client);
frame[client_offset..client_offset + 16].copy_from_slice(&0xCAFE_u128.to_le_bytes());
frame[offset_of!(PrepareHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE;
- // Seal the checksum the way a real sender does, so the frame's
- // only anomaly is the operation byte itself.
+ // A prepare's `checksum` carries the identity checksum, computed
+ // over the operation byte among others; stamping it the way a real
+ // sender does leaves the unknown byte as the frame's only anomaly
+ // and is what lets the classifier trust that byte.
let header: &[u8; HEADER_SIZE] = frame[..HEADER_SIZE]
.try_into()
.expect("frame spans a full header");
- let checksum = frame_checksum_bytes(header);
- frame[..size_of::<u128>()].copy_from_slice(&checksum.to_le_bytes());
+ let identity = prepare_identity_checksum_bytes(header);
+ frame[..size_of::<u128>()].copy_from_slice(&identity.to_le_bytes());
}
// The generic view carries no operation field, so the receive path
@@ -858,7 +885,12 @@ mod tests {
};
assert!(
- !matches!(error, ConsensusError::InvalidBitPattern),
+ matches!(
+ error,
+ ConsensusError::UnsupportedOperation {
+ operation: OPERATION_FROM_A_NEWER_RELEASE
+ }
+ ),
"unknown operation {OPERATION_FROM_A_NEWER_RELEASE:#x} is silently dropped: the \
typed decode collapses a wire-valid frame from a newer release into the same \
InvalidBitPattern as corruption, and dispatch drops both with only a warn log, \