You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/03/31 13:09:56 UTC
(iggy) branch master updated: feat(partitions): rework partitions using new message (#3020)
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 0935a76eb feat(partitions): rework partitions using new message (#3020)
0935a76eb is described below
commit 0935a76eb59920ea512063c2b8d2a14fc74945c1
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Tue Mar 31 15:09:53 2026 +0200
feat(partitions): rework partitions using new message (#3020)
Refactors partition messaging around Message<H, B>, where the backing
storage is part of the type rather than
being encoded through the old TwoHalves prefix/tail layout.
Request messages now use a contiguous Owned backing, while
response-style paths can use fragmented storage
directly. This removes the old prefix/tail copy-on-write design
entirely, along with the aliasing and uniqueness
problems it introduced. In particular, storage reads no longer need to
reconstruct a request-shaped message from
a split buffer, which was fragile once the tail could already be shared
by cache-backed Frozen values. The new
design makes the ownership model explicit, keeps the request path
simple, and avoids relying on prefix
detachment semantics to preserve correctness.
---------
Co-authored-by: Hubert Gruszecki <h....@gmail.com>
---
Cargo.lock | 16 +-
Cargo.toml | 1 +
core/binary_protocol/Cargo.toml | 2 +
core/binary_protocol/src/consensus/message.rs | 660 ++++++++-------
core/binary_protocol/src/consensus/mod.rs | 7 +-
core/binary_protocol/src/lib.rs | 8 +-
core/common/Cargo.toml | 3 +
core/common/src/error/iggy_error.rs | 4 +
core/common/src/lib.rs | 2 +
core/common/src/types/mod.rs | 1 +
.../src/types/segment_storage/messages_writer.rs | 5 +-
core/common/src/types/send_messages2.rs | 755 +++++++++++++++++
core/consensus/Cargo.toml | 2 +
core/consensus/src/impls.rs | 360 ++++++--
core/consensus/src/lib.rs | 23 +-
core/consensus/src/namespaced_pipeline.rs | 50 +-
core/consensus/src/observability.rs | 604 ++++++++++++++
core/consensus/src/plane_helpers.rs | 122 +--
core/consensus/src/plane_mux.rs | 12 +-
core/iobuf/Cargo.toml | 1 +
core/iobuf/src/lib.rs | 647 ++++++---------
core/journal/Cargo.toml | 2 +-
core/journal/src/metadata_journal.rs | 43 +-
core/message_bus/src/lib.rs | 16 +-
core/metadata/Cargo.toml | 1 +
core/metadata/src/impls/metadata.rs | 171 ++--
core/metadata/src/impls/recovery.rs | 10 +-
core/metadata/src/stm/mod.rs | 8 +-
core/partitions/Cargo.toml | 4 +
core/partitions/src/iggy_index.rs | 164 ++++
core/partitions/src/iggy_index_writer.rs | 113 +++
core/partitions/src/iggy_partition.rs | 181 ++--
core/partitions/src/iggy_partitions.rs | 916 +++++++++++++++------
core/partitions/src/journal.rs | 367 ++++++---
core/partitions/src/lib.rs | 49 +-
core/partitions/src/log.rs | 57 +-
.../src}/messages_writer.rs | 99 +--
core/partitions/src/segment.rs | 85 ++
.../mod.rs => partitions/src/send_messages2.rs} | 23 -
core/partitions/src/types.rs | 65 +-
core/shard/src/lib.rs | 1 +
core/shard/src/router.rs | 36 +-
core/simulator/Cargo.toml | 1 +
core/simulator/src/bus.rs | 26 +-
core/simulator/src/client.rs | 116 +--
core/simulator/src/deps.rs | 20 +-
core/simulator/src/lib.rs | 21 +-
core/simulator/src/main.rs | 17 +-
core/simulator/src/packet.rs | 22 +-
49 files changed, 4271 insertions(+), 1648 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index f8b5a94d0..6322dd011 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2233,9 +2233,11 @@ dependencies = [
"futures",
"iggy_binary_protocol",
"iggy_common",
+ "iobuf",
"message_bus",
"rand 0.10.0",
"rand_xoshiro",
+ "tracing",
]
[[package]]
@@ -5390,6 +5392,8 @@ dependencies = [
"bytemuck",
"bytes",
"enumset",
+ "iobuf",
+ "smallvec",
"thiserror 2.0.18",
]
@@ -5406,6 +5410,7 @@ dependencies = [
"blake3",
"bon",
"byte-unit",
+ "bytemuck",
"bytes",
"chrono",
"clap",
@@ -5416,6 +5421,8 @@ dependencies = [
"err_trail",
"human-repr",
"humantime",
+ "iggy_binary_protocol",
+ "iobuf",
"lending-iterator",
"moka",
"nix",
@@ -5911,6 +5918,7 @@ name = "iobuf"
version = "0.1.0"
dependencies = [
"aligned-vec",
+ "compio-buf",
]
[[package]]
@@ -6049,9 +6057,9 @@ name = "journal"
version = "0.1.0"
dependencies = [
"bytemuck",
- "bytes",
"compio",
"iggy_binary_protocol",
+ "iobuf",
"tempfile",
]
@@ -6671,6 +6679,7 @@ dependencies = [
"consensus",
"iggy_binary_protocol",
"iggy_common",
+ "iobuf",
"journal",
"left-right",
"message_bus",
@@ -7646,13 +7655,17 @@ dependencies = [
name = "partitions"
version = "0.1.0"
dependencies = [
+ "bytemuck",
"bytes",
+ "compio",
"consensus",
"iggy_binary_protocol",
"iggy_common",
+ "iobuf",
"journal",
"message_bus",
"ringbuffer",
+ "smallvec",
"tokio",
"tracing",
]
@@ -9946,6 +9959,7 @@ dependencies = [
"futures",
"iggy_binary_protocol",
"iggy_common",
+ "iobuf",
"journal",
"message_bus",
"metadata",
diff --git a/Cargo.toml b/Cargo.toml
index ed714d5cd..c14d9f2e2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -166,6 +166,7 @@ iggy_binary_protocol = { path = "core/binary_protocol", version = "0.9.5-edge.1"
iggy_common = { path = "core/common", version = "0.9.5-edge.1" }
iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.2.2-edge.1" }
integration = { path = "core/integration" }
+iobuf = { path = "core/iobuf" }
journal = { path = "core/journal" }
js-sys = "0.3"
jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] }
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml
index 51f310908..b8d82e430 100644
--- a/core/binary_protocol/Cargo.toml
+++ b/core/binary_protocol/Cargo.toml
@@ -32,6 +32,8 @@ readme = "../../README.md"
bytemuck = { workspace = true }
bytes = { workspace = true }
enumset = { workspace = true }
+iobuf = { workspace = true }
+smallvec = "1.13"
thiserror = { workspace = true }
[lints.clippy]
diff --git a/core/binary_protocol/src/consensus/message.rs b/core/binary_protocol/src/consensus/message.rs
index f63cf1c19..834394f33 100644
--- a/core/binary_protocol/src/consensus/message.rs
+++ b/core/binary_protocol/src/consensus/message.rs
@@ -15,88 +15,277 @@
// specific language governing permissions and limitations
// under the License.
-//! Zero-copy consensus message wrapper.
-//!
-//! `Message<H>` wraps a `Bytes` buffer and provides typed access to the
-//! header via `bytemuck` pointer cast (zero allocation, zero copy).
-
use crate::consensus::{
- Command2, ConsensusError, ConsensusHeader, GenericHeader, PrepareHeader, PrepareOkHeader,
+ self, Command2, ConsensusError, ConsensusHeader, GenericHeader, PrepareHeader, PrepareOkHeader,
RequestHeader,
};
-use bytes::Bytes;
-use std::marker::PhantomData;
+use iobuf::{Frozen, Owned};
+use smallvec::SmallVec;
+use std::{marker::PhantomData, mem::size_of};
+
+const MESSAGE_ALIGN: usize = 4096;
-/// Trait for types that provide access to a consensus header.
-pub trait ConsensusMessage<H: ConsensusHeader> {
+pub trait MessageBacking<H>
+where
+ H: ConsensusHeader,
+{
fn header(&self) -> &H;
+ fn header_storage(&self) -> &[u8];
+ fn total_len(&self) -> usize;
+}
+
+pub trait RequestBackingKind {}
+pub trait ResponseBackingKind {}
+
+pub trait MutableBacking<H>: MessageBacking<H> + RequestBackingKind
+where
+ H: ConsensusHeader,
+{
+ fn as_slice(&self) -> &[u8];
+ fn as_mut_slice(&mut self) -> &mut [u8];
+}
+
+pub trait FragmentedBacking<H>: MessageBacking<H> + ResponseBackingKind
+where
+ H: ConsensusHeader,
+{
+ fn fragments(&self) -> &[Frozen<MESSAGE_ALIGN>];
+}
+
+#[derive(Debug)]
+pub struct RequestBacking {
+ owned: Owned<MESSAGE_ALIGN>,
+}
+
+#[derive(Debug, Clone)]
+pub struct ResponseBacking {
+ fragments: SmallVec<[Frozen<MESSAGE_ALIGN>; 4]>,
}
-impl<H: ConsensusHeader> ConsensusMessage<H> for Message<H> {
+impl RequestBackingKind for RequestBacking {}
+impl ResponseBackingKind for ResponseBacking {}
+
+impl RequestBacking {
+ fn into_owned(self) -> Owned<MESSAGE_ALIGN> {
+ self.owned
+ }
+
+ fn into_frozen(self) -> Frozen<MESSAGE_ALIGN> {
+ self.owned.into()
+ }
+}
+
+impl<H> MessageBacking<H> for RequestBacking
+where
+ H: ConsensusHeader,
+{
fn header(&self) -> &H {
- let header_bytes = &self.buffer[..size_of::<H>()];
- bytemuck::checked::try_from_bytes(header_bytes)
- .expect("header validated at construction time")
+ let bytes = &self.owned.as_slice()[..size_of::<H>()];
+ bytemuck::checked::try_from_bytes(bytes)
+ .expect("header bytes must match the requested header type")
+ }
+
+ fn header_storage(&self) -> &[u8] {
+ self.owned.as_slice()
+ }
+
+ fn total_len(&self) -> usize {
+ self.owned.as_slice().len()
}
}
-/// Zero-copy message wrapping a `Bytes` buffer with a typed header.
-///
-/// The header is accessed via `bytemuck::try_from_bytes` - a pointer cast,
-/// not a deserialization step. The body is the slice after the header,
-/// sized according to `header.size()`.
+impl<H> MutableBacking<H> for RequestBacking
+where
+ H: ConsensusHeader,
+{
+ fn as_slice(&self) -> &[u8] {
+ self.owned.as_slice()
+ }
+
+ fn as_mut_slice(&mut self) -> &mut [u8] {
+ self.owned.as_mut_slice()
+ }
+}
+
+impl<H> MessageBacking<H> for ResponseBacking
+where
+ H: ConsensusHeader,
+{
+ fn header(&self) -> &H {
+ let first = self
+ .fragments
+ .first()
+ .expect("response backing validated at construction time");
+ let bytes = &first.as_slice()[..size_of::<H>()];
+ bytemuck::checked::try_from_bytes(bytes)
+ .expect("response header bytes must match the requested header type")
+ }
+
+ fn header_storage(&self) -> &[u8] {
+ self.fragments
+ .first()
+ .expect("response backing validated at construction time")
+ .as_slice()
+ }
+
+ fn total_len(&self) -> usize {
+ self.fragments.iter().map(Frozen::len).sum()
+ }
+}
+
+impl<H> FragmentedBacking<H> for ResponseBacking
+where
+ H: ConsensusHeader,
+{
+ fn fragments(&self) -> &[Frozen<MESSAGE_ALIGN>] {
+ &self.fragments
+ }
+}
+
+pub trait ConsensusMessage<H>
+where
+ H: ConsensusHeader,
+{
+ fn header(&self) -> &H;
+}
+
+impl<H, B> ConsensusMessage<H> for Message<H, B>
+where
+ H: ConsensusHeader,
+ B: MessageBacking<H>,
+{
+ fn header(&self) -> &H {
+ self.backing.header()
+ }
+}
+
+#[derive(Debug)]
#[repr(C)]
-#[derive(Debug, Clone)]
-pub struct Message<H: ConsensusHeader> {
- buffer: Bytes,
+pub struct Message<H, B = RequestBacking> {
+ backing: B,
_marker: PhantomData<H>,
}
-impl<H: ConsensusHeader> Message<H> {
- /// Create a message from a buffer, validating the header.
- ///
+impl<H, B> Message<H, B>
+where
+ H: ConsensusHeader,
+ B: MessageBacking<H>,
+{
+ pub fn header(&self) -> &H {
+ self.backing.header()
+ }
+
+ pub fn total_len(&self) -> usize {
+ self.backing.total_len()
+ }
+
+ pub fn into_inner(self) -> B {
+ self.backing
+ }
+
+ pub fn into_generic(self) -> Message<GenericHeader, B>
+ where
+ B: MessageBacking<GenericHeader>,
+ {
+ Message {
+ backing: self.backing,
+ _marker: PhantomData,
+ }
+ }
+
+ pub const fn as_generic(&self) -> &Message<GenericHeader, B>
+ where
+ B: MessageBacking<GenericHeader>,
+ {
+ unsafe { &*std::ptr::from_ref(self).cast::<Message<GenericHeader, B>>() }
+ }
+
/// # Errors
- /// Returns `ConsensusError` if the buffer is too small or the header is invalid.
- pub fn from_bytes(buffer: Bytes) -> Result<Self, ConsensusError> {
- if buffer.len() < size_of::<H>() {
+ ///
+ /// Returns [`ConsensusError`] if the backing is too short for `T`, the
+ /// command encoded in the generic header does not match `T::COMMAND`, or
+ /// the typed header fails validation.
+ pub fn try_into_typed<T>(self) -> Result<Message<T, B>, ConsensusError>
+ where
+ T: ConsensusHeader,
+ B: MessageBacking<GenericHeader> + MessageBacking<T>,
+ {
+ if self.total_len() < size_of::<T>() {
return Err(ConsensusError::InvalidCommand {
- expected: H::COMMAND,
+ expected: T::COMMAND,
found: Command2::Reserved,
});
}
- // TODO(hubcio): bytemuck::checked::try_from_bytes requires the buffer to be aligned
- // to the target type's alignment. Consensus headers contain u128 fields (16-byte
- // alignment), but Bytes from Vec<u8> only guarantees 8-byte alignment. The checked
- // variant returns Err on misalignment (no UB), but production code can fail on
- // misaligned buffers. The original iggy_common code has the same limitation.
- // Fix by either using pod_read_unaligned (copies header) or ensuring aligned
- // allocation at the network receive layer.
- let header_bytes = &buffer[..size_of::<H>()];
- let header = bytemuck::checked::try_from_bytes::<H>(header_bytes)
+ let generic = self.as_generic();
+ if generic.header().command != T::COMMAND {
+ return Err(ConsensusError::InvalidCommand {
+ expected: T::COMMAND,
+ found: generic.header().command,
+ });
+ }
+
+ let bytes = <B as MessageBacking<T>>::header_storage(&self.backing);
+ let typed = bytemuck::checked::try_from_bytes::<T>(&bytes[..size_of::<T>()])
.map_err(|_| ConsensusError::InvalidBitPattern)?;
+ typed.validate()?;
- header.validate()?;
+ Ok(Message {
+ backing: self.backing,
+ _marker: PhantomData,
+ })
+ }
- let header_size = header.size() as usize;
- if buffer.len() < header_size {
+ /// # Errors
+ ///
+ /// Returns [`ConsensusError`] if the backing is too short for `T`, the
+ /// command encoded in the generic header does not match `T::COMMAND`, or
+ /// the typed header fails validation.
+ pub fn try_as_typed<T>(&self) -> Result<&Message<T, B>, ConsensusError>
+ where
+ T: ConsensusHeader,
+ B: MessageBacking<GenericHeader> + MessageBacking<T>,
+ {
+ if self.total_len() < size_of::<T>() {
return Err(ConsensusError::InvalidCommand {
- expected: H::COMMAND,
+ expected: T::COMMAND,
found: Command2::Reserved,
});
}
- Ok(Self {
- buffer,
+ let generic = self.as_generic();
+ if generic.header().command != T::COMMAND {
+ return Err(ConsensusError::InvalidCommand {
+ expected: T::COMMAND,
+ found: generic.header().command,
+ });
+ }
+
+ let bytes = <B as MessageBacking<T>>::header_storage(&self.backing);
+ let typed = bytemuck::checked::try_from_bytes::<T>(&bytes[..size_of::<T>()])
+ .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ typed.validate()?;
+
+ let typed_message = unsafe { &*std::ptr::from_ref(self).cast::<Message<T, B>>() };
+ let _ = typed;
+ Ok(typed_message)
+ }
+
+ const unsafe fn from_backing_unchecked(backing: B) -> Self {
+ Self {
+ backing,
_marker: PhantomData,
- })
+ }
}
+}
- /// Create a zero-initialized message of the given size.
- /// The caller must initialize the header fields.
- ///
+impl<H> Message<H>
+where
+ H: ConsensusHeader,
+{
/// # Panics
- /// Panics if `size` is smaller than the header size.
+ ///
+ /// Panics if `size` is smaller than `size_of::<H>()`.
#[must_use]
pub fn new(size: usize) -> Self {
assert!(
@@ -104,189 +293,154 @@ impl<H: ConsensusHeader> Message<H> {
"size must be at least header size ({})",
size_of::<H>()
);
- Self {
- buffer: Bytes::from(vec![0u8; size]),
- _marker: PhantomData,
+
+ unsafe {
+ Self::from_backing_unchecked(RequestBacking {
+ owned: Owned::<MESSAGE_ALIGN>::zeroed(size),
+ })
}
}
- /// Access the typed header (zero-copy pointer cast).
- ///
- /// Re-validates the bit pattern on each call because headers contain
- /// `CheckedBitPattern` enums (`Command2`, `Operation`) which prevent
- /// using the unchecked `bytemuck::try_from_bytes` (requires `Pod`).
- /// The cost is a few enum discriminant range checks per access.
- /// The buffer is immutable (`Bytes`) so validation cannot fail after
- /// successful construction.
#[must_use]
- #[allow(clippy::missing_panics_doc)]
- #[inline]
- pub fn header(&self) -> &H {
- let header_bytes = &self.buffer[..size_of::<H>()];
- bytemuck::checked::try_from_bytes(header_bytes)
- .expect("header validated at construction time")
+ pub fn as_slice(&self) -> &[u8] {
+ <RequestBacking as MutableBacking<H>>::as_slice(&self.backing)
}
- /// Message body (everything after the header).
- #[must_use]
- #[inline]
- pub fn body(&self) -> &[u8] {
- let header_size = size_of::<H>();
- let total_size = self.header().size() as usize;
- if total_size > header_size {
- &self.buffer[header_size..total_size]
- } else {
- &[]
- }
+ pub fn as_mut_slice(&mut self) -> &mut [u8] {
+ <RequestBacking as MutableBacking<H>>::as_mut_slice(&mut self.backing)
}
- /// Message body as zero-copy [`Bytes`].
- #[must_use]
- #[inline]
- pub fn body_bytes(&self) -> Bytes {
- let header_size = size_of::<H>();
- let total_size = self.header().size() as usize;
- if total_size > header_size {
- self.buffer.slice(header_size..total_size)
- } else {
- Bytes::new()
- }
+ pub fn prefix_mut(&mut self) -> &mut [u8] {
+ self.as_mut_slice()
}
- /// Complete message bytes (header + body).
+ /// # Panics
+ ///
+ /// Panics if re-validating the copied message unexpectedly fails.
#[must_use]
- #[inline]
- pub fn as_bytes(&self) -> &[u8] {
- let total_size = self.header().size() as usize;
- &self.buffer[..total_size]
+ pub fn deep_copy(&self) -> Self {
+ Self::try_from(Owned::<MESSAGE_ALIGN>::copy_from_slice(self.as_slice()))
+ .expect("deep copied request message must stay valid")
}
- /// Consume into the underlying buffer.
#[must_use]
- #[inline]
- pub fn into_inner(self) -> Bytes {
- self.buffer
+ pub fn into_owned(self) -> Owned<MESSAGE_ALIGN> {
+ self.backing.into_owned()
}
- /// Convert to a generic message (type-erase the header).
#[must_use]
- pub fn into_generic(self) -> Message<GenericHeader> {
- // SAFETY: Message<H> and Message<GenericHeader> have identical layout
- // (only PhantomData differs). GenericHeader accepts any command.
- unsafe { Message::from_buffer_unchecked(self.buffer) }
+ pub fn into_frozen(self) -> Frozen<MESSAGE_ALIGN> {
+ self.backing.into_frozen()
+ }
+
+ /// # Panics
+ ///
+ /// Panics if `H` and `T` have different sizes, or if the rewritten header
+ /// does not validate as `T`.
+ pub fn transmute_header<T: ConsensusHeader>(self, f: impl FnOnce(H, &mut T)) -> Message<T> {
+ assert_eq!(size_of::<H>(), size_of::<T>());
+
+ let old_header = *self.header();
+ let mut owned = self.into_owned();
+ let slice = &mut owned.as_mut_slice()[..size_of::<T>()];
+ slice.fill(0);
+ let new_header =
+ bytemuck::checked::try_from_bytes_mut(slice).expect("zeroed bytes are valid");
+ f(old_header, new_header);
+
+ Message::try_from(owned).expect("transmuted request message must stay valid")
}
+}
- /// View as a generic message without consuming.
+impl<H> Message<H, ResponseBacking>
+where
+ H: ConsensusHeader,
+{
#[must_use]
- #[allow(clippy::missing_const_for_fn)]
- pub fn as_generic(&self) -> &Message<GenericHeader> {
- // SAFETY: #[repr(C)] guarantees field layout. Message<H> and
- // Message<GenericHeader> differ only in PhantomData (ZST).
- unsafe { &*std::ptr::from_ref(self).cast::<Message<GenericHeader>>() }
+ pub fn fragments(&self) -> &[Frozen<MESSAGE_ALIGN>] {
+ <ResponseBacking as FragmentedBacking<H>>::fragments(&self.backing)
}
+}
- /// Try to reinterpret as a different header type.
- ///
- /// # Errors
- /// Returns `ConsensusError` if the header command doesn't match or validation fails.
- pub fn try_into_typed<T: ConsensusHeader>(self) -> Result<Message<T>, ConsensusError> {
- if self.buffer.len() < size_of::<T>() {
- return Err(ConsensusError::InvalidCommand {
- expected: T::COMMAND,
- found: Command2::Reserved,
- });
+impl<H> Clone for Message<H, ResponseBacking>
+where
+ H: ConsensusHeader,
+{
+ fn clone(&self) -> Self {
+ Self {
+ backing: self.backing.clone(),
+ _marker: PhantomData,
}
+ }
+}
- let generic = self.as_generic();
- if generic.header().command != T::COMMAND {
+impl<H> TryFrom<Owned<MESSAGE_ALIGN>> for Message<H>
+where
+ H: ConsensusHeader,
+{
+ type Error = ConsensusError;
+
+ fn try_from(owned: Owned<MESSAGE_ALIGN>) -> Result<Self, Self::Error> {
+ let bytes = owned.as_slice();
+ if bytes.len() < size_of::<H>() {
return Err(ConsensusError::InvalidCommand {
- expected: T::COMMAND,
- found: generic.header().command,
+ expected: H::COMMAND,
+ found: Command2::Reserved,
});
}
- let header_bytes = &self.buffer[..size_of::<T>()];
- let header = bytemuck::checked::try_from_bytes::<T>(header_bytes)
+ let header = bytemuck::checked::try_from_bytes::<H>(&bytes[..size_of::<H>()])
.map_err(|_| ConsensusError::InvalidBitPattern)?;
header.validate()?;
- Ok(unsafe { Message::<T>::from_buffer_unchecked(self.buffer) })
+ if bytes.len() < header.size() as usize {
+ return Err(ConsensusError::InvalidCommand {
+ expected: H::COMMAND,
+ found: Command2::Reserved,
+ });
+ }
+
+ Ok(unsafe { Self::from_backing_unchecked(RequestBacking { owned }) })
}
+}
- /// Try to borrow as a different header type without consuming.
- ///
- /// # Errors
- /// Returns `ConsensusError` if the header command doesn't match or validation fails.
- pub fn try_as_typed<T: ConsensusHeader>(&self) -> Result<&Message<T>, ConsensusError> {
- if self.buffer.len() < size_of::<T>() {
+impl<H> TryFrom<SmallVec<[Frozen<MESSAGE_ALIGN>; 4]>> for Message<H, ResponseBacking>
+where
+ H: ConsensusHeader,
+{
+ type Error = ConsensusError;
+
+ fn try_from(fragments: SmallVec<[Frozen<MESSAGE_ALIGN>; 4]>) -> Result<Self, Self::Error> {
+ let Some(first) = fragments.first() else {
return Err(ConsensusError::InvalidCommand {
- expected: T::COMMAND,
+ expected: H::COMMAND,
found: Command2::Reserved,
});
- }
+ };
- let generic = self.as_generic();
- if generic.header().command != T::COMMAND {
+ if first.len() < size_of::<H>() {
return Err(ConsensusError::InvalidCommand {
- expected: T::COMMAND,
- found: generic.header().command,
+ expected: H::COMMAND,
+ found: Command2::Reserved,
});
}
- let header_bytes = &self.buffer[..size_of::<T>()];
- bytemuck::checked::try_from_bytes::<T>(header_bytes)
- .map_err(|_| ConsensusError::InvalidBitPattern)?
- .validate()?;
-
- // SAFETY: Message<H> and Message<T> have identical layout (#[repr(C)],
- // differ only in PhantomData). Header validated above.
- Ok(unsafe { &*std::ptr::from_ref(self).cast::<Message<T>>() })
- }
+ let header = bytemuck::checked::try_from_bytes::<H>(&first.as_slice()[..size_of::<H>()])
+ .map_err(|_| ConsensusError::InvalidBitPattern)?;
+ header.validate()?;
- /// Transmute the header to a different type, preserving the buffer.
- ///
- /// The callback receives the old header (by value) and a mutable reference
- /// to the new (zeroed) header, allowing field-by-field initialization.
- ///
- /// # Panics
- /// Panics if `size_of::<H>() != size_of::<T>()`.
- #[must_use]
- pub fn transmute_header<T: ConsensusHeader>(self, f: impl FnOnce(H, &mut T)) -> Message<T> {
- assert_eq!(size_of::<H>(), size_of::<T>());
- let old_header = *self.header();
- let buffer = self.into_inner();
- // TODO: This mutates through Bytes via cast_mut(), which violates aliasing rules.
- // The original iggy_common code has the same pattern. Once iggy_common is migrated
- // to use iggy_binary_protocol types, fix both by using Bytes::try_into_mut() ->
- // mutate BytesMut -> freeze(). Tracked as part of the consensus migration.
- unsafe {
- let ptr = buffer.as_ptr().cast_mut();
- let slice = std::slice::from_raw_parts_mut(ptr, size_of::<T>());
- slice.fill(0);
- let new_header =
- bytemuck::checked::try_from_bytes_mut(slice).expect("zeroed bytes are valid");
- f(old_header, new_header);
- }
- Message {
- buffer,
- _marker: PhantomData,
+ let total_len = fragments.iter().map(Frozen::len).sum::<usize>();
+ if total_len < header.size() as usize {
+ return Err(ConsensusError::InvalidCommand {
+ expected: H::COMMAND,
+ found: Command2::Reserved,
+ });
}
- }
- /// Create without validation. Private.
- ///
- /// # Safety
- /// Buffer must already be validated for the target header type.
- #[inline]
- const unsafe fn from_buffer_unchecked(buffer: Bytes) -> Self {
- Self {
- buffer,
- _marker: PhantomData,
- }
+ Ok(unsafe { Self::from_backing_unchecked(ResponseBacking { fragments }) })
}
}
-/// Type-erased message bag for dispatching incoming consensus messages.
#[derive(Debug)]
pub enum MessageBag {
Request(Message<RequestHeader>),
@@ -298,49 +452,52 @@ impl MessageBag {
#[must_use]
pub fn command(&self) -> Command2 {
match self {
- Self::Request(m) => m.header().command,
- Self::Prepare(m) => m.header().command,
- Self::PrepareOk(m) => m.header().command,
+ Self::Request(message) => message.header().command,
+ Self::Prepare(message) => message.header().command,
+ Self::PrepareOk(message) => message.header().command,
}
}
#[must_use]
pub fn size(&self) -> u32 {
match self {
- Self::Request(m) => m.header().size(),
- Self::Prepare(m) => m.header().size(),
- Self::PrepareOk(m) => m.header().size(),
+ Self::Request(message) => message.header().size(),
+ Self::Prepare(message) => message.header().size(),
+ Self::PrepareOk(message) => message.header().size(),
}
}
#[must_use]
- pub fn operation(&self) -> crate::consensus::Operation {
+ pub fn operation(&self) -> consensus::Operation {
match self {
- Self::Request(m) => m.header().operation,
- Self::Prepare(m) => m.header().operation,
- Self::PrepareOk(m) => m.header().operation,
+ Self::Request(message) => message.header().operation,
+ Self::Prepare(message) => message.header().operation,
+ Self::PrepareOk(message) => message.header().operation,
}
}
}
-impl<T: ConsensusHeader> TryFrom<Message<T>> for MessageBag {
+impl<T> TryFrom<Message<T>> for MessageBag
+where
+ T: ConsensusHeader,
+{
type Error = ConsensusError;
fn try_from(value: Message<T>) -> Result<Self, Self::Error> {
let command = value.as_generic().header().command;
- let buffer = value.into_inner();
+ let backing = value.into_inner();
match command {
- Command2::Request => {
- let msg = unsafe { Message::<RequestHeader>::from_buffer_unchecked(buffer) };
- Ok(Self::Request(msg))
- }
Command2::Prepare => {
- let msg = unsafe { Message::<PrepareHeader>::from_buffer_unchecked(buffer) };
+ let msg = unsafe { Message::<PrepareHeader>::from_backing_unchecked(backing) };
Ok(Self::Prepare(msg))
}
+ Command2::Request => {
+ let msg = unsafe { Message::<RequestHeader>::from_backing_unchecked(backing) };
+ Ok(Self::Request(msg))
+ }
Command2::PrepareOk => {
- let msg = unsafe { Message::<PrepareOkHeader>::from_buffer_unchecked(buffer) };
+ let msg = unsafe { Message::<PrepareOkHeader>::from_backing_unchecked(backing) };
Ok(Self::PrepareOk(msg))
}
other => Err(ConsensusError::InvalidCommand {
@@ -350,100 +507,3 @@ impl<T: ConsensusHeader> TryFrom<Message<T>> for MessageBag {
}
}
}
-
-#[cfg(test)]
-mod tests {
- use super::Message;
- use crate::consensus::{Command2, GenericHeader, RequestHeader};
- use bytes::{Bytes, BytesMut};
-
- #[test]
- fn generic_message_from_zeroed_buffer() {
- let mut buf = BytesMut::zeroed(384);
- let header =
- bytemuck::checked::try_from_bytes_mut::<GenericHeader>(&mut buf[..256]).unwrap();
- header.size = 384;
- header.cluster = 42;
-
- let msg = Message::<GenericHeader>::from_bytes(buf.freeze()).unwrap();
- assert_eq!(msg.header().cluster, 42);
- assert_eq!(msg.header().size, 384);
- assert_eq!(msg.body().len(), 128);
- }
-
- #[test]
- fn request_message_roundtrip() {
- let mut buf = BytesMut::zeroed(320);
- let header =
- bytemuck::checked::try_from_bytes_mut::<RequestHeader>(&mut buf[..256]).unwrap();
- header.size = 320;
- header.command = Command2::Request;
- header.client = 12345;
- header.operation = crate::consensus::Operation::CreateStream;
-
- let msg = Message::<RequestHeader>::from_bytes(buf.freeze()).unwrap();
- assert_eq!(msg.header().client, 12345);
- assert_eq!(
- msg.header().operation,
- crate::consensus::Operation::CreateStream
- );
- assert_eq!(msg.body().len(), 64);
- }
-
- #[test]
- fn too_small_buffer_rejected() {
- let buf = Bytes::from(vec![0u8; 100]); // < 256
- assert!(Message::<GenericHeader>::from_bytes(buf).is_err());
- }
-
- #[test]
- fn into_generic_and_back() {
- let mut buf = BytesMut::zeroed(256);
- let header =
- bytemuck::checked::try_from_bytes_mut::<RequestHeader>(&mut buf[..256]).unwrap();
- header.size = 256;
- header.command = Command2::Request;
-
- let msg = Message::<RequestHeader>::from_bytes(buf.freeze()).unwrap();
- let generic = msg.into_generic();
- assert_eq!(generic.header().command, Command2::Request);
-
- let back = generic.try_into_typed::<RequestHeader>().unwrap();
- assert_eq!(back.header().command, Command2::Request);
- }
-
- #[test]
- fn transmute_header_generic_to_request() {
- let mut buf = BytesMut::zeroed(320);
- let header =
- bytemuck::checked::try_from_bytes_mut::<GenericHeader>(&mut buf[..256]).unwrap();
- header.size = 320;
- header.cluster = 99;
-
- let generic = Message::<GenericHeader>::from_bytes(buf.freeze()).unwrap();
- let request = generic.transmute_header(|old, new: &mut RequestHeader| {
- new.size = old.size;
- new.command = Command2::Request;
- new.cluster = old.cluster;
- new.client = 42;
- new.operation = crate::consensus::Operation::CreateStream;
- });
- assert_eq!(request.header().command, Command2::Request);
- assert_eq!(request.header().client, 42);
- assert_eq!(request.header().cluster, 99);
- assert_eq!(request.header().size, 320);
- assert_eq!(request.body().len(), 64);
- }
-
- #[test]
- fn wrong_type_conversion_fails() {
- let mut buf = BytesMut::zeroed(256);
- let header =
- bytemuck::checked::try_from_bytes_mut::<GenericHeader>(&mut buf[..256]).unwrap();
- header.size = 256;
- header.command = Command2::Reserved; // not Request
-
- let msg = Message::<GenericHeader>::from_bytes(buf.freeze()).unwrap();
- assert!(msg.try_into_typed::<RequestHeader>().is_err());
- }
-}
diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs
index 3840eec82..0801dbbc3 100644
--- a/core/binary_protocol/src/consensus/mod.rs
+++ b/core/binary_protocol/src/consensus/mod.rs
@@ -37,7 +37,7 @@
//! - [`StartViewHeader`] - new primary -> all replicas (header-only)
//!
//! ## Message wrapper
-//! - [`message::Message`] - zero-copy `Bytes` wrapper with typed header access
+//! - [`message::Message`] - typed consensus message wrapper over request or response backing
mod command;
mod error;
@@ -51,5 +51,8 @@ pub use header::{
CommitHeader, ConsensusHeader, DoViewChangeHeader, GenericHeader, HEADER_SIZE, PrepareHeader,
PrepareOkHeader, ReplyHeader, RequestHeader, StartViewChangeHeader, StartViewHeader,
};
-pub use message::{ConsensusMessage, MessageBag};
+pub use message::{
+ ConsensusMessage, FragmentedBacking, Message, MessageBacking, MessageBag, MutableBacking,
+ RequestBacking, RequestBackingKind, ResponseBacking, ResponseBackingKind,
+};
pub use operation::Operation;
diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs
index cce1e5d56..8e5541516 100644
--- a/core/binary_protocol/src/lib.rs
+++ b/core/binary_protocol/src/lib.rs
@@ -46,7 +46,7 @@
//!
//! All consensus headers are 256 bytes with `#[repr(C)]` layout.
//! Deserialization is zero-copy via `bytemuck`. The [`Message`] type
-//! wraps a `Bytes` buffer with typed header access.
+//! wraps request or response backing with typed header access.
//!
//! - Client-facing: [`RequestHeader`], [`ReplyHeader`]
//! - Replication: [`PrepareHeader`], [`PrepareOkHeader`], [`CommitHeader`]
@@ -69,8 +69,10 @@ pub mod responses;
pub use codec::{WireDecode, WireEncode};
pub use consensus::{
Command2, CommitHeader, ConsensusError, ConsensusHeader, ConsensusMessage, DoViewChangeHeader,
- GenericHeader, HEADER_SIZE, MessageBag, Operation, PrepareHeader, PrepareOkHeader, ReplyHeader,
- RequestHeader, StartViewChangeHeader, StartViewHeader, message::Message,
+ FragmentedBacking, GenericHeader, HEADER_SIZE, Message, MessageBacking, MessageBag,
+ MutableBacking, Operation, PrepareHeader, PrepareOkHeader, ReplyHeader, RequestBacking,
+ RequestBackingKind, RequestHeader, ResponseBacking, ResponseBackingKind, StartViewChangeHeader,
+ StartViewHeader,
};
pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command};
pub use error::WireError;
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml
index a2fd8ae21..3a89f6cd3 100644
--- a/core/common/Cargo.toml
+++ b/core/common/Cargo.toml
@@ -37,6 +37,7 @@ base64 = { workspace = true }
blake3 = { workspace = true }
bon = { workspace = true }
byte-unit = { workspace = true }
+bytemuck = { workspace = true }
bytes = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true }
@@ -47,6 +48,8 @@ derive_more = { workspace = true }
err_trail = { workspace = true }
human-repr = { workspace = true }
humantime = { workspace = true }
+iggy_binary_protocol = { workspace = true }
+iobuf = { workspace = true }
lending-iterator = { workspace = true }
moka = { workspace = true }
once_cell = { workspace = true }
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index 18d759c79..dca0f75c2 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -384,6 +384,10 @@ pub enum IggyError {
InvalidMessagesSize(u32, u32) = 4036,
#[error("Too small message: {0}B, expected: {1}B")]
TooSmallMessage(u32, u32) = 4037,
+ #[error("Invalid message timestamp delta: {0} microseconds exceeds the per-batch limit")]
+ InvalidMessageTimestampDelta(u64) = 4038,
+ #[error("Invalid batch checksum: {0}, expected: {1}, for base offset: {2}")]
+ InvalidBatchChecksum(u64, u64, u64) = 4039,
#[error("Cannot sed messages due to client disconnection")]
CannotSendMessagesDueToClientDisconnection = 4050,
#[error("Background send error")]
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs
index c690b5a1c..f27289a3b 100644
--- a/core/common/src/lib.rs
+++ b/core/common/src/lib.rs
@@ -113,6 +113,8 @@ pub use types::permissions::personal_access_token::*;
pub use types::personal_access_tokens::*;
pub use types::segment::Segment;
pub use types::segment_storage::*;
+pub use types::send_messages2;
+pub use types::send_messages2::*;
pub use types::snapshot::*;
pub use types::stats::*;
pub use types::stream::*;
diff --git a/core/common/src/types/mod.rs b/core/common/src/types/mod.rs
index 01b19de7b..06ab6914a 100644
--- a/core/common/src/types/mod.rs
+++ b/core/common/src/types/mod.rs
@@ -32,6 +32,7 @@ pub(crate) mod permissions;
pub(crate) mod personal_access_tokens;
pub(crate) mod segment;
pub(crate) mod segment_storage;
+pub mod send_messages2;
pub(crate) mod snapshot;
pub(crate) mod stats;
pub(crate) mod stream;
diff --git a/core/common/src/types/segment_storage/messages_writer.rs b/core/common/src/types/segment_storage/messages_writer.rs
index 6f81f7752..051fc3f1b 100644
--- a/core/common/src/types/segment_storage/messages_writer.rs
+++ b/core/common/src/types/segment_storage/messages_writer.rs
@@ -117,11 +117,14 @@ impl MessagesWriter {
Ok(IggyByteSize::from(messages_size))
}
-
pub fn path(&self) -> String {
self.file_path.clone()
}
+ pub fn size_counter(&self) -> Rc<AtomicU64> {
+ self.messages_size_bytes.clone()
+ }
+
pub async fn fsync(&self) -> Result<(), IggyError> {
self.file
.sync_all()
diff --git a/core/common/src/types/send_messages2.rs b/core/common/src/types/send_messages2.rs
new file mode 100644
index 000000000..e714936da
--- /dev/null
+++ b/core/common/src/types/send_messages2.rs
@@ -0,0 +1,755 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{INDEX_SIZE, IggyError, random_id, sharding::IggyNamespace};
+use bytes::{Bytes, BytesMut};
+use iggy_binary_protocol::{Message, PrepareHeader, RequestHeader};
+use iobuf::Owned;
+use std::hash::Hasher;
+use twox_hash::XxHash3_64;
+
+const MESSAGE_ALIGN: usize = 4096;
+pub const COMMAND_HEADER_SIZE: usize = 256;
+pub const PREPARE_SPLIT_POINT: usize = 512;
+const MESSAGE_HEADER_SIZE: usize = 48;
+const LEGACY_MESSAGE_HEADER_SIZE: usize = 64;
+const BATCH_CHECKSUM_OFFSET: usize = 40;
+const MESSAGE_COUNT_OFFSET: usize = 48;
+const MAX_TIMESTAMP_DELTA_MICROS: u64 = u32::MAX as u64;
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct SendMessages2Header {
+ pub partition_id: u64,
+ pub base_offset: u64,
+ pub base_timestamp: u64,
+ pub origin_timestamp: u64,
+ pub batch_length: u64,
+ pub batch_checksum: u64,
+ pub message_count: u32,
+}
+
+impl SendMessages2Header {
+ pub const fn new(
+ partition_id: u64,
+ origin_timestamp: u64,
+ batch_length: u64,
+ message_count: u32,
+ ) -> Self {
+ Self {
+ partition_id,
+ base_offset: 0,
+ base_timestamp: 0,
+ origin_timestamp,
+ batch_length,
+ batch_checksum: 0,
+ message_count,
+ }
+ }
+
+ pub fn decode(bytes: &[u8]) -> Result<Self, IggyError> {
+ if bytes.len() < COMMAND_HEADER_SIZE {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let batch_length = read_u64(bytes, 32)?;
+ if batch_length < COMMAND_HEADER_SIZE as u64 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(Self {
+ partition_id: read_u64(bytes, 0)?,
+ base_offset: read_u64(bytes, 8)?,
+ base_timestamp: read_u64(bytes, 16)?,
+ origin_timestamp: read_u64(bytes, 24)?,
+ batch_length,
+ batch_checksum: read_u64(bytes, BATCH_CHECKSUM_OFFSET)?,
+ message_count: read_u32(bytes, MESSAGE_COUNT_OFFSET)?,
+ })
+ }
+
+ pub fn encode_into(&self, bytes: &mut [u8]) {
+ assert!(bytes.len() >= COMMAND_HEADER_SIZE);
+ bytes[..COMMAND_HEADER_SIZE].fill(0);
+ bytes[0..8].copy_from_slice(&self.partition_id.to_le_bytes());
+ bytes[8..16].copy_from_slice(&self.base_offset.to_le_bytes());
+ bytes[16..24].copy_from_slice(&self.base_timestamp.to_le_bytes());
+ bytes[24..32].copy_from_slice(&self.origin_timestamp.to_le_bytes());
+ bytes[32..40].copy_from_slice(&self.batch_length.to_le_bytes());
+ bytes[BATCH_CHECKSUM_OFFSET..BATCH_CHECKSUM_OFFSET + 8]
+ .copy_from_slice(&self.batch_checksum.to_le_bytes());
+ bytes[MESSAGE_COUNT_OFFSET..MESSAGE_COUNT_OFFSET + 4]
+ .copy_from_slice(&self.message_count.to_le_bytes());
+ }
+
+ pub fn total_size(&self) -> usize {
+ usize::try_from(self.batch_length).expect("batch length exceeds usize::MAX")
+ }
+
+ pub fn blob_len(&self) -> Result<usize, IggyError> {
+ usize::try_from(
+ self.batch_length
+ .checked_sub(COMMAND_HEADER_SIZE as u64)
+ .ok_or(IggyError::InvalidCommand)?,
+ )
+ .map_err(|_| IggyError::InvalidCommand)
+ }
+
+ pub fn into_frozen(self) -> FrozenBatchHeader {
+ let mut buffer = Owned::<MESSAGE_ALIGN>::zeroed(COMMAND_HEADER_SIZE);
+ self.encode_into(buffer.as_mut_slice());
+ buffer.into()
+ }
+
+ #[must_use]
+ pub fn checksum_for_blob(&self, blob: &[u8]) -> u64 {
+ calculate_batch_checksum(self, blob)
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct SendMessages2Owned {
+ pub header: SendMessages2Header,
+ pub blob: Bytes,
+}
+
+impl SendMessages2Owned {
+ pub fn from_messages(
+ namespace: IggyNamespace,
+ messages: &IggyMessages2,
+ ) -> Result<Self, IggyError> {
+ let message_count = messages.count();
+ let mut origin_timestamp = u64::MAX;
+ for message in messages {
+ origin_timestamp = origin_timestamp.min(message.header.origin_timestamp);
+ }
+
+ if origin_timestamp == u64::MAX {
+ origin_timestamp = 0;
+ }
+
+ let mut blob = BytesMut::new();
+ for (index, message) in messages.iter().enumerate() {
+ let id = if message.header.id == 0 {
+ random_id::get_uuid()
+ } else {
+ message.header.id
+ };
+ let offset_delta = u32::try_from(index).map_err(|_| IggyError::InvalidCommand)?;
+ let timestamp_delta = message
+ .header
+ .origin_timestamp
+ .checked_sub(origin_timestamp)
+ .ok_or(IggyError::InvalidCommand)?;
+ if timestamp_delta > MAX_TIMESTAMP_DELTA_MICROS {
+ return Err(IggyError::InvalidMessageTimestampDelta(timestamp_delta));
+ }
+ let timestamp_delta =
+ u32::try_from(timestamp_delta).map_err(|_| IggyError::InvalidCommand)?;
+ let user_headers = message.user_headers.as_deref().unwrap_or_default();
+ let user_headers_length =
+ u32::try_from(user_headers.len()).map_err(|_| IggyError::InvalidCommand)?;
+ let payload_length =
+ u32::try_from(message.payload.len()).map_err(|_| IggyError::InvalidCommand)?;
+
+ let mut header = [0u8; MESSAGE_HEADER_SIZE];
+ header[8..24].copy_from_slice(&id.to_le_bytes());
+ header[24..28].copy_from_slice(&offset_delta.to_le_bytes());
+ header[28..32].copy_from_slice(×tamp_delta.to_le_bytes());
+ header[32..36].copy_from_slice(&user_headers_length.to_le_bytes());
+ header[36..40].copy_from_slice(&payload_length.to_le_bytes());
+
+ let checksum = calculate_checksum_parts(&header[8..], user_headers, &message.payload);
+ header[0..8].copy_from_slice(&checksum.to_le_bytes());
+
+ blob.extend_from_slice(&header);
+ blob.extend_from_slice(user_headers);
+ blob.extend_from_slice(&message.payload);
+ }
+
+ let blob = blob.freeze();
+ let mut header = SendMessages2Header::new(
+ namespace.partition_id() as u64,
+ origin_timestamp,
+ u64::try_from(COMMAND_HEADER_SIZE + blob.len())
+ .map_err(|_| IggyError::InvalidCommand)?,
+ message_count,
+ );
+ header.batch_checksum = calculate_batch_checksum(&header, &blob);
+
+ Ok(Self { header, blob })
+ }
+
+ pub fn from_legacy_request(namespace: IggyNamespace, body: &[u8]) -> Result<Self, IggyError> {
+ let (message_count, messages) = legacy_messages_slice(body)?;
+ let mut parsed = Vec::with_capacity(message_count as usize);
+ let mut origin_timestamp = u64::MAX;
+ let mut cursor = 0usize;
+
+ while cursor < messages.len() && parsed.len() < message_count as usize {
+ let legacy = LegacyMessageRef::decode(&messages[cursor..])?;
+ origin_timestamp = origin_timestamp.min(legacy.origin_timestamp);
+ cursor += legacy.total_size;
+ parsed.push(legacy);
+ }
+
+ if parsed.len() != message_count as usize || cursor != messages.len() {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ if origin_timestamp == u64::MAX {
+ origin_timestamp = 0;
+ }
+
+ let mut blob = BytesMut::with_capacity(messages.len());
+ for (index, legacy) in parsed.iter().enumerate() {
+ let id = if legacy.id == 0 {
+ random_id::get_uuid()
+ } else {
+ legacy.id
+ };
+ let offset_delta = u32::try_from(index).map_err(|_| IggyError::InvalidCommand)?;
+ let timestamp_delta = legacy
+ .origin_timestamp
+ .checked_sub(origin_timestamp)
+ .ok_or(IggyError::InvalidCommand)?;
+ if timestamp_delta > MAX_TIMESTAMP_DELTA_MICROS {
+ return Err(IggyError::InvalidMessageTimestampDelta(timestamp_delta));
+ }
+ let timestamp_delta =
+ u32::try_from(timestamp_delta).map_err(|_| IggyError::InvalidCommand)?;
+ let user_headers_length =
+ u32::try_from(legacy.user_headers.len()).map_err(|_| IggyError::InvalidCommand)?;
+ let payload_length =
+ u32::try_from(legacy.payload.len()).map_err(|_| IggyError::InvalidCommand)?;
+
+ let mut header = [0u8; MESSAGE_HEADER_SIZE];
+ header[8..24].copy_from_slice(&id.to_le_bytes());
+ header[24..28].copy_from_slice(&offset_delta.to_le_bytes());
+ header[28..32].copy_from_slice(×tamp_delta.to_le_bytes());
+ header[32..36].copy_from_slice(&user_headers_length.to_le_bytes());
+ header[36..40].copy_from_slice(&payload_length.to_le_bytes());
+
+ let checksum =
+ calculate_checksum_parts(&header[8..], legacy.user_headers, legacy.payload);
+ header[0..8].copy_from_slice(&checksum.to_le_bytes());
+
+ blob.extend_from_slice(&header);
+ blob.extend_from_slice(legacy.user_headers);
+ blob.extend_from_slice(legacy.payload);
+ }
+
+ let blob = blob.freeze();
+ let mut header = SendMessages2Header::new(
+ namespace.partition_id() as u64,
+ origin_timestamp,
+ u64::try_from(COMMAND_HEADER_SIZE + blob.len())
+ .map_err(|_| IggyError::InvalidCommand)?,
+ message_count,
+ );
+ header.batch_checksum = calculate_batch_checksum(&header, &blob);
+
+ Ok(Self { header, blob })
+ }
+
+ pub fn encode_request(
+ self,
+ request_header: RequestHeader,
+ ) -> Result<Message<RequestHeader>, IggyError> {
+ let total_size = std::mem::size_of::<RequestHeader>() + self.header.total_size();
+ let mut buffer = Owned::<MESSAGE_ALIGN>::zeroed(total_size);
+ let bytes = buffer.as_mut_slice();
+ bytes[0..std::mem::size_of::<RequestHeader>()]
+ .copy_from_slice(bytemuck::bytes_of(&request_header));
+ self.header.encode_into(
+ &mut bytes[std::mem::size_of::<RequestHeader>()
+ ..std::mem::size_of::<RequestHeader>() + COMMAND_HEADER_SIZE],
+ );
+ bytes[PREPARE_SPLIT_POINT..PREPARE_SPLIT_POINT + self.blob.len()]
+ .copy_from_slice(&self.blob);
+
+ Message::try_from(buffer).map_err(|_| IggyError::InvalidCommand)
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct IggyMessage2Header {
+ pub checksum: u64,
+ pub id: u128,
+ pub offset: u64,
+ pub timestamp: u64,
+ pub origin_timestamp: u64,
+ pub user_headers_length: u32,
+ pub payload_length: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IggyMessage2 {
+ pub header: IggyMessage2Header,
+ pub payload: Bytes,
+ pub user_headers: Option<Bytes>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct IggyMessages2 {
+ messages: Vec<IggyMessage2>,
+}
+
+impl IggyMessages2 {
+ #[must_use]
+ pub fn empty() -> Self {
+ Self::default()
+ }
+
+ #[must_use]
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ messages: Vec::with_capacity(capacity),
+ }
+ }
+
+ pub fn push(&mut self, message: IggyMessage2) {
+ self.messages.push(message);
+ }
+
+ #[must_use]
+ pub fn count(&self) -> u32 {
+ u32::try_from(self.messages.len()).unwrap_or(u32::MAX)
+ }
+
+ #[must_use]
+ pub const fn is_empty(&self) -> bool {
+ self.messages.is_empty()
+ }
+
+ #[must_use]
+ pub fn first_offset(&self) -> Option<u64> {
+ self.messages.first().map(|message| message.header.offset)
+ }
+
+ #[must_use]
+ pub fn last_offset(&self) -> Option<u64> {
+ self.messages.last().map(|message| message.header.offset)
+ }
+
+ #[must_use]
+ pub fn limit(self, count: u32) -> Self {
+ let mut messages = self.messages;
+ messages.truncate(usize::try_from(count).unwrap_or(usize::MAX));
+ Self { messages }
+ }
+
+ pub fn iter(&self) -> std::slice::Iter<'_, IggyMessage2> {
+ self.messages.iter()
+ }
+}
+
+impl IntoIterator for IggyMessages2 {
+ type Item = IggyMessage2;
+ type IntoIter = std::vec::IntoIter<IggyMessage2>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.messages.into_iter()
+ }
+}
+
+impl<'a> IntoIterator for &'a IggyMessages2 {
+ type Item = &'a IggyMessage2;
+ type IntoIter = std::slice::Iter<'a, IggyMessage2>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.messages.iter()
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct SendMessages2Ref<'a> {
+ pub header: SendMessages2Header,
+ blob: &'a [u8],
+}
+
+#[allow(dead_code)]
+impl<'a> SendMessages2Ref<'a> {
+ pub const fn iter(&self) -> SendMessages2Iterator<'a> {
+ SendMessages2Iterator {
+ blob: self.blob,
+ position: 0,
+ }
+ }
+
+ pub const fn iter_with_offsets(&self) -> SendMessages2IteratorWithOffsets<'a> {
+ SendMessages2IteratorWithOffsets {
+ blob: self.blob,
+ position: 0,
+ }
+ }
+
+ pub const fn blob(&self) -> &'a [u8] {
+ self.blob
+ }
+
+ pub const fn message_count(&self) -> u32 {
+ self.header.message_count
+ }
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+pub struct SendMessages2MessageHeader {
+ pub checksum: u64,
+ pub id: u128,
+ pub offset_delta: u32,
+ /// Microsecond delta from `SendMessages2Header::origin_timestamp`.
+ ///
+ /// This is stored in `u32`, which limits a single batch to roughly
+ /// 71.6 minutes of origin timestamp span.
+ pub timestamp_delta: u32,
+ pub user_headers_length: u32,
+ pub payload_length: u32,
+}
+
+impl SendMessages2MessageHeader {
+ fn decode(bytes: &[u8]) -> Result<Self, IggyError> {
+ if bytes.len() < MESSAGE_HEADER_SIZE {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let reserved = read_u64(bytes, 40)?;
+ if reserved != 0 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(Self {
+ checksum: read_u64(bytes, 0)?,
+ id: read_u128(bytes, 8)?,
+ offset_delta: read_u32(bytes, 24)?,
+ timestamp_delta: read_u32(bytes, 28)?,
+ user_headers_length: read_u32(bytes, 32)?,
+ payload_length: read_u32(bytes, 36)?,
+ })
+ }
+
+ const fn total_size(&self) -> usize {
+ MESSAGE_HEADER_SIZE + self.user_headers_length as usize + self.payload_length as usize
+ }
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+pub struct SendMessages2MessageView<'a> {
+ pub header: SendMessages2MessageHeader,
+ pub user_headers: &'a [u8],
+ pub payload: &'a [u8],
+}
+
+#[allow(dead_code)]
+impl SendMessages2MessageView<'_> {
+ pub fn owned_message(&self, batch: &SendMessages2Header) -> IggyMessage2 {
+ IggyMessage2 {
+ header: IggyMessage2Header {
+ checksum: self.header.checksum,
+ id: self.header.id,
+ offset: batch.base_offset + u64::from(self.header.offset_delta),
+ timestamp: batch.base_timestamp,
+ origin_timestamp: batch.origin_timestamp + u64::from(self.header.timestamp_delta),
+ user_headers_length: self.header.user_headers_length,
+ payload_length: self.header.payload_length,
+ },
+ payload: Bytes::copy_from_slice(self.payload),
+ user_headers: (!self.user_headers.is_empty())
+ .then(|| Bytes::copy_from_slice(self.user_headers)),
+ }
+ }
+}
+
+#[allow(dead_code)]
+pub struct SendMessages2Iterator<'a> {
+ blob: &'a [u8],
+ position: usize,
+}
+
+impl<'a> Iterator for SendMessages2Iterator<'a> {
+ type Item = SendMessages2MessageView<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.position >= self.blob.len() {
+ return None;
+ }
+
+ let header = SendMessages2MessageHeader::decode(&self.blob[self.position..]).ok()?;
+ let start = self.position + MESSAGE_HEADER_SIZE;
+ let headers_end = start + header.user_headers_length as usize;
+ let payload_end = headers_end + header.payload_length as usize;
+ let user_headers = self.blob.get(start..headers_end)?;
+ let payload = self.blob.get(headers_end..payload_end)?;
+ self.position += header.total_size();
+ Some(SendMessages2MessageView {
+ header,
+ user_headers,
+ payload,
+ })
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct SendMessages2MessageViewWithOffsets<'a> {
+ pub message: SendMessages2MessageView<'a>,
+ pub start: usize,
+ pub end: usize,
+}
+
+pub struct SendMessages2IteratorWithOffsets<'a> {
+ blob: &'a [u8],
+ position: usize,
+}
+
+impl<'a> Iterator for SendMessages2IteratorWithOffsets<'a> {
+ type Item = SendMessages2MessageViewWithOffsets<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.position >= self.blob.len() {
+ return None;
+ }
+
+ let start = self.position;
+ let header = SendMessages2MessageHeader::decode(&self.blob[self.position..]).ok()?;
+ let message_start = self.position + MESSAGE_HEADER_SIZE;
+ let headers_end = message_start + header.user_headers_length as usize;
+ let payload_end = headers_end + header.payload_length as usize;
+ let user_headers = self.blob.get(message_start..headers_end)?;
+ let payload = self.blob.get(headers_end..payload_end)?;
+ self.position += header.total_size();
+ Some(SendMessages2MessageViewWithOffsets {
+ message: SendMessages2MessageView {
+ header,
+ user_headers,
+ payload,
+ },
+ start,
+ end: self.position,
+ })
+ }
+}
+
+pub type FrozenBatchHeader = iobuf::Frozen<MESSAGE_ALIGN>;
+
+pub fn convert_request_message(
+ namespace: IggyNamespace,
+ message: Message<RequestHeader>,
+) -> Result<Message<RequestHeader>, IggyError> {
+ let request_header = *message.header();
+ let total_size = request_header.size as usize;
+ let body = &message.as_slice()[std::mem::size_of::<RequestHeader>()..total_size];
+ if decode_request_slice(body).is_ok() {
+ return Ok(message);
+ }
+ SendMessages2Owned::from_legacy_request(namespace, body)?.encode_request(request_header)
+}
+
+fn decode_request_slice(body: &[u8]) -> Result<SendMessages2Ref<'_>, IggyError> {
+ if body.len() < COMMAND_HEADER_SIZE {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let header = SendMessages2Header::decode(&body[..COMMAND_HEADER_SIZE])?;
+ let blob_len = header.blob_len()?;
+ if body.len() < header.total_size() {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let blob = &body[COMMAND_HEADER_SIZE..COMMAND_HEADER_SIZE + blob_len];
+ let expected_checksum = calculate_batch_checksum(&header, blob);
+ if header.batch_checksum != expected_checksum {
+ return Err(IggyError::InvalidBatchChecksum(
+ header.batch_checksum,
+ expected_checksum,
+ header.base_offset,
+ ));
+ }
+
+ Ok(SendMessages2Ref { header, blob })
+}
+
+pub fn decode_prepare_slice(bytes: &[u8]) -> Result<SendMessages2Ref<'_>, IggyError> {
+ let header_size = std::mem::size_of::<PrepareHeader>();
+ if bytes.len() < header_size {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let prepare = bytemuck::checked::try_from_bytes::<PrepareHeader>(&bytes[..header_size])
+ .map_err(|_| IggyError::InvalidCommand)?;
+ let total_size = prepare.size as usize;
+ if bytes.len() < total_size {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let body = &bytes[header_size..total_size];
+ if body.len() < COMMAND_HEADER_SIZE {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let header = SendMessages2Header::decode(&body[..COMMAND_HEADER_SIZE])?;
+ let blob = &body[COMMAND_HEADER_SIZE..];
+ let blob_len = header.blob_len()?;
+ if body.len() < header.total_size() {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let blob = &blob[..blob_len];
+ let expected_checksum = calculate_batch_checksum(&header, blob);
+ if header.batch_checksum != expected_checksum {
+ return Err(IggyError::InvalidBatchChecksum(
+ header.batch_checksum,
+ expected_checksum,
+ header.base_offset,
+ ));
+ }
+
+ Ok(SendMessages2Ref { header, blob })
+}
+
+pub fn stamp_prepare_for_persistence(
+ mut message: Message<PrepareHeader>,
+ base_offset: u64,
+ base_timestamp: u64,
+) -> Result<(Message<PrepareHeader>, SendMessages2Header, u32), IggyError> {
+ let total_size = message.header().size as usize;
+ let bytes = message.as_mut_slice();
+ if bytes.len() < PREPARE_SPLIT_POINT || total_size < PREPARE_SPLIT_POINT {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let header_offset = std::mem::size_of::<PrepareHeader>();
+ let mut command =
+ SendMessages2Header::decode(&bytes[header_offset..header_offset + COMMAND_HEADER_SIZE])?;
+ command.base_offset = base_offset;
+ command.base_timestamp = base_timestamp;
+ let blob = &bytes[PREPARE_SPLIT_POINT..total_size];
+ command.batch_checksum = calculate_batch_checksum(&command, blob);
+ command.encode_into(&mut bytes[header_offset..header_offset + COMMAND_HEADER_SIZE]);
+ Ok((message, command, command.message_count))
+}
+
+fn legacy_messages_slice(body: &[u8]) -> Result<(u32, &[u8]), IggyError> {
+ if body.len() < 4 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let metadata_length = read_u32(body, 0)? as usize;
+ let metadata_end = 4usize
+ .checked_add(metadata_length)
+ .ok_or(IggyError::InvalidCommand)?;
+ if metadata_end < 4 || body.len() < metadata_end {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let message_count = read_u32(body, metadata_end - 4)?;
+ let indexes_len = usize::try_from(message_count)
+ .ok()
+ .and_then(|count| count.checked_mul(INDEX_SIZE))
+ .ok_or(IggyError::InvalidCommand)?;
+ let messages_start = metadata_end
+ .checked_add(indexes_len)
+ .ok_or(IggyError::InvalidCommand)?;
+ if body.len() < messages_start {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok((message_count, &body[messages_start..]))
+}
+
+#[derive(Clone, Copy)]
+struct LegacyMessageRef<'a> {
+ id: u128,
+ origin_timestamp: u64,
+ user_headers: &'a [u8],
+ payload: &'a [u8],
+ total_size: usize,
+}
+
+impl<'a> LegacyMessageRef<'a> {
+ fn decode(bytes: &'a [u8]) -> Result<Self, IggyError> {
+ if bytes.len() < LEGACY_MESSAGE_HEADER_SIZE {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let user_headers_length = read_u32(bytes, 48)? as usize;
+ let payload_length = read_u32(bytes, 52)? as usize;
+ let total_size = LEGACY_MESSAGE_HEADER_SIZE
+ .checked_add(payload_length)
+ .and_then(|size| size.checked_add(user_headers_length))
+ .ok_or(IggyError::InvalidCommand)?;
+ if bytes.len() < total_size {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let payload_start = LEGACY_MESSAGE_HEADER_SIZE;
+ let payload_end = payload_start + payload_length;
+ let headers_end = payload_end + user_headers_length;
+
+ Ok(Self {
+ id: read_u128(bytes, 8)?,
+ origin_timestamp: read_u64(bytes, 40)?,
+ user_headers: &bytes[payload_end..headers_end],
+ payload: &bytes[payload_start..payload_end],
+ total_size,
+ })
+ }
+}
+
+fn calculate_checksum_parts(header_tail: &[u8], user_headers: &[u8], payload: &[u8]) -> u64 {
+ let mut hasher = XxHash3_64::new();
+ hasher.write(header_tail);
+ hasher.write(user_headers);
+ hasher.write(payload);
+ hasher.finish()
+}
+
+fn calculate_batch_checksum(header: &SendMessages2Header, blob: &[u8]) -> u64 {
+ let mut hasher = XxHash3_64::new();
+ hasher.write(&header.partition_id.to_le_bytes());
+ hasher.write(&header.base_offset.to_le_bytes());
+ hasher.write(&header.base_timestamp.to_le_bytes());
+ hasher.write(&header.origin_timestamp.to_le_bytes());
+ hasher.write(&header.batch_length.to_le_bytes());
+ hasher.write(&header.message_count.to_le_bytes());
+ hasher.write(blob);
+ hasher.finish()
+}
+
+fn read_u32(bytes: &[u8], offset: usize) -> Result<u32, IggyError> {
+ bytes
+ .get(offset..offset + 4)
+ .and_then(|slice| slice.try_into().ok())
+ .map(u32::from_le_bytes)
+ .ok_or(IggyError::InvalidNumberEncoding)
+}
+
+fn read_u64(bytes: &[u8], offset: usize) -> Result<u64, IggyError> {
+ bytes
+ .get(offset..offset + 8)
+ .and_then(|slice| slice.try_into().ok())
+ .map(u64::from_le_bytes)
+ .ok_or(IggyError::InvalidNumberEncoding)
+}
+
+fn read_u128(bytes: &[u8], offset: usize) -> Result<u128, IggyError> {
+ bytes
+ .get(offset..offset + 16)
+ .and_then(|slice| slice.try_into().ok())
+ .map(u128::from_le_bytes)
+ .ok_or(IggyError::InvalidNumberEncoding)
+}
diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml
index ddf7a6764..dac219e3a 100644
--- a/core/consensus/Cargo.toml
+++ b/core/consensus/Cargo.toml
@@ -33,9 +33,11 @@ bytemuck = { workspace = true }
bytes = { workspace = true }
iggy_binary_protocol = { workspace = true }
iggy_common = { workspace = true }
+iobuf = { workspace = true }
message_bus = { workspace = true }
rand = { workspace = true }
rand_xoshiro = { workspace = true }
+tracing = { workspace = true }
[dev-dependencies]
futures = { workspace = true }
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index ea2ac7c61..7c4b27d04 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -17,8 +17,10 @@
use crate::vsr_timeout::{TimeoutKind, TimeoutManager};
use crate::{
- Consensus, DvcQuorumArray, Pipeline, Project, StoredDvc, dvc_count, dvc_max_commit,
- dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_select_winner,
+ AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, IgnoreReason, Pipeline,
+ PlaneKind, PrepareLogEvent, Project, ReplicaLogContext, SimEventKind, StoredDvc,
+ ViewChangeLogEvent, ViewChangeReason, dvc_count, dvc_max_commit, dvc_quorum_array_empty,
+ dvc_record, dvc_reset, dvc_select_winner, emit_replica_event, emit_sim_event,
};
use bit_set::BitSet;
use iggy_binary_protocol::{
@@ -131,6 +133,7 @@ pub struct RequestEntry {
}
impl RequestEntry {
+ #[must_use]
pub const fn new(message: Message<RequestHeader>) -> Self {
Self {
message,
@@ -180,17 +183,15 @@ impl LocalPipeline {
self.prepare_queue.is_empty()
}
- /// Push a new message to the pipeline.
+ /// Push a new entry to the pipeline.
///
/// # Panics
/// - If message queue is full.
/// - If the message doesn't chain correctly to the previous entry.
- // Signature must match `Pipeline::push_message` trait definition.
- #[allow(clippy::needless_pass_by_value)]
- pub fn push_message(&mut self, message: Message<PrepareHeader>) {
+ pub fn push(&mut self, entry: PipelineEntry) {
assert!(!self.prepare_queue_full(), "prepare queue is full");
- let header = *message.header();
+ let header = entry.header;
if let Some(tail) = self.prepare_queue.back() {
let tail_header = &tail.header;
@@ -208,7 +209,12 @@ impl LocalPipeline {
assert!(header.view >= tail_header.view, "view cannot go backwards");
}
- self.prepare_queue.push_back(PipelineEntry::new(header));
+ self.prepare_queue.push_back(entry);
+ }
+
+ #[allow(clippy::needless_pass_by_value)]
+ pub fn push_message(&mut self, message: Message<PrepareHeader>) {
+ self.push(PipelineEntry::new(*message.header()));
}
/// Pop the oldest message (after it's been committed).
@@ -342,14 +348,13 @@ impl LocalPipeline {
}
impl Pipeline for LocalPipeline {
- type Message = Message<PrepareHeader>;
type Entry = PipelineEntry;
- fn push_message(&mut self, message: Self::Message) {
- Self::push_message(self, message);
+ fn push(&mut self, entry: Self::Entry) {
+ Self::push(self, entry);
}
- fn pop_message(&mut self) -> Option<Self::Entry> {
+ fn pop(&mut self) -> Option<Self::Entry> {
Self::pop_message(self)
}
@@ -357,15 +362,15 @@ impl Pipeline for LocalPipeline {
Self::clear(self);
}
- fn message_by_op(&self, op: u64) -> Option<&Self::Entry> {
+ fn entry_by_op(&self, op: u64) -> Option<&Self::Entry> {
Self::message_by_op(self, op)
}
- fn message_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry> {
+ fn entry_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry> {
Self::message_by_op_mut(self, op)
}
- fn message_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry> {
+ fn entry_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry> {
Self::message_by_op_and_checksum(self, op, checksum)
}
@@ -381,6 +386,10 @@ impl Pipeline for LocalPipeline {
Self::is_empty(self)
}
+ fn len(&self) -> usize {
+ self.prepare_count()
+ }
+
fn verify(&self) {
Self::verify(self);
}
@@ -423,6 +432,27 @@ pub enum VsrAction {
},
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PrepareOkOutcome {
+ Accepted {
+ ack_count: usize,
+ quorum_reached: bool,
+ },
+ Ignored {
+ reason: IgnoreReason,
+ },
+}
+
+impl PrepareOkOutcome {
+ #[must_use]
+ pub const fn quorum_reached(self) -> bool {
+ match self {
+ Self::Accepted { quorum_reached, .. } => quorum_reached,
+ Self::Ignored { .. } => false,
+ }
+ }
+}
+
#[allow(unused)]
#[derive(Debug)]
pub struct VsrConsensus<B = IggyMessageBus, P = LocalPipeline>
@@ -565,12 +595,6 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
self.commit.get()
}
- #[must_use]
- pub const fn is_syncing(&self) -> bool {
- // TODO: for now return false. we have to add syncing related setup to VsrConsensus to make this work.
- false
- }
-
#[must_use]
pub const fn replica(&self) -> u8 {
self.replica
@@ -688,7 +712,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// Returns a list of actions to take based on fired timeouts.
/// Empty vec means no actions needed.
- pub fn tick(&self, current_op: u64, current_commit: u64) -> Vec<VsrAction> {
+ pub fn tick(&self, plane: PlaneKind, current_op: u64, current_commit: u64) -> Vec<VsrAction> {
let mut actions = Vec::new();
let mut timeouts = self.timeouts.borrow_mut();
@@ -698,25 +722,29 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// Phase 2: Handle fired timeouts
if timeouts.fired(TimeoutKind::NormalHeartbeat) {
drop(timeouts);
- actions.extend(self.handle_normal_heartbeat_timeout());
+ actions.extend(self.handle_normal_heartbeat_timeout(plane));
timeouts = self.timeouts.borrow_mut();
}
if timeouts.fired(TimeoutKind::StartViewChangeMessage) {
drop(timeouts);
- actions.extend(self.handle_start_view_change_message_timeout());
+ actions.extend(self.handle_start_view_change_message_timeout(plane));
timeouts = self.timeouts.borrow_mut();
}
if timeouts.fired(TimeoutKind::DoViewChangeMessage) {
drop(timeouts);
- actions.extend(self.handle_do_view_change_message_timeout(current_op, current_commit));
+ actions.extend(self.handle_do_view_change_message_timeout(
+ plane,
+ current_op,
+ current_commit,
+ ));
timeouts = self.timeouts.borrow_mut();
}
if timeouts.fired(TimeoutKind::ViewChangeStatus) {
drop(timeouts);
- actions.extend(self.handle_view_change_status_timeout());
+ actions.extend(self.handle_view_change_status_timeout(plane));
// timeouts = self.timeouts.borrow_mut(); // Not needed if last
}
@@ -725,7 +753,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
/// Called when `normal_heartbeat` timeout fires.
/// Backup hasn't heard from primary - start view change.
- fn handle_normal_heartbeat_timeout(&self) -> Vec<VsrAction> {
+ fn handle_normal_heartbeat_timeout(&self, plane: PlaneKind) -> Vec<VsrAction> {
// Only backups trigger view change on heartbeat timeout
if self.is_primary() {
return Vec::new();
@@ -737,7 +765,8 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
}
// Advance to new view and transition to view change
- let new_view = self.view.get() + 1;
+ let old_view = self.view.get();
+ let new_view = old_view + 1;
self.view.set(new_view);
self.status.set(Status::ViewChange);
@@ -755,14 +784,32 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
timeouts.start(TimeoutKind::ViewChangeStatus);
}
- vec![VsrAction::SendStartViewChange {
+ emit_sim_event(
+ SimEventKind::ViewChangeStarted,
+ &ViewChangeLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ old_view,
+ new_view,
+ reason: ViewChangeReason::NormalHeartbeatTimeout,
+ },
+ );
+
+ let action = VsrAction::SendStartViewChange {
view: new_view,
namespace: self.namespace,
- }]
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ vec![action]
}
/// Resend SVC message if we've started view change.
- fn handle_start_view_change_message_timeout(&self) -> Vec<VsrAction> {
+ fn handle_start_view_change_message_timeout(&self, plane: PlaneKind) -> Vec<VsrAction> {
if !self.sent_own_start_view_change.get() {
return Vec::new();
}
@@ -771,15 +818,24 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
.borrow_mut()
.reset(TimeoutKind::StartViewChangeMessage);
- vec![VsrAction::SendStartViewChange {
+ let action = VsrAction::SendStartViewChange {
view: self.view.get(),
namespace: self.namespace,
- }]
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ vec![action]
}
/// Resend DVC message if we've sent one.
fn handle_do_view_change_message_timeout(
&self,
+ plane: PlaneKind,
current_op: u64,
current_commit: u64,
) -> Vec<VsrAction> {
@@ -800,24 +856,33 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
.borrow_mut()
.reset(TimeoutKind::DoViewChangeMessage);
- vec![VsrAction::SendDoViewChange {
+ let action = VsrAction::SendDoViewChange {
view: self.view.get(),
target: self.primary_index(self.view.get()),
log_view: self.log_view.get(),
op: current_op,
commit: current_commit,
namespace: self.namespace,
- }]
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ vec![action]
}
/// Escalate to next view if stuck in view change.
- fn handle_view_change_status_timeout(&self) -> Vec<VsrAction> {
+ fn handle_view_change_status_timeout(&self, plane: PlaneKind) -> Vec<VsrAction> {
if self.status.get() != Status::ViewChange {
return Vec::new();
}
// Escalate: try next view
- let next_view = self.view.get() + 1;
+ let old_view = self.view.get();
+ let next_view = old_view + 1;
self.view.set(next_view);
self.reset_view_change_state();
@@ -830,10 +895,28 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
.borrow_mut()
.reset(TimeoutKind::ViewChangeStatus);
- vec![VsrAction::SendStartViewChange {
+ emit_sim_event(
+ SimEventKind::ViewChangeStarted,
+ &ViewChangeLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ old_view,
+ new_view: next_view,
+ reason: ViewChangeReason::ViewChangeStatusTimeout,
+ },
+ );
+
+ let action = VsrAction::SendStartViewChange {
view: next_view,
namespace: self.namespace,
- }]
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ vec![action]
}
/// Handle a received `StartViewChange` message.
@@ -844,7 +927,11 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// # Panics
/// If `header.namespace` does not match this replica's namespace.
- pub fn handle_start_view_change(&self, header: &StartViewChangeHeader) -> Vec<VsrAction> {
+ pub fn handle_start_view_change(
+ &self,
+ plane: PlaneKind,
+ header: &StartViewChangeHeader,
+ ) -> Vec<VsrAction> {
assert_eq!(
header.namespace, self.namespace,
"SVC routed to wrong group"
@@ -861,6 +948,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// If SVC is for a higher view, advance to that view
if msg_view > self.view.get() {
+ let old_view = self.view.get();
self.view.set(msg_view);
self.status.set(Status::ViewChange);
self.reset_view_change_state();
@@ -877,11 +965,29 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
timeouts.start(TimeoutKind::ViewChangeStatus);
}
+ emit_sim_event(
+ SimEventKind::ViewChangeStarted,
+ &ViewChangeLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ old_view,
+ new_view: msg_view,
+ reason: ViewChangeReason::ReceivedStartViewChange,
+ },
+ );
+
// Send our own SVC
- actions.push(VsrAction::SendStartViewChange {
+ let action = VsrAction::SendStartViewChange {
view: msg_view,
namespace: self.namespace,
- });
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ actions.push(action);
}
// Record the SVC from sender
@@ -905,14 +1011,22 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
.borrow_mut()
.start(TimeoutKind::DoViewChangeMessage);
- actions.push(VsrAction::SendDoViewChange {
+ let action = VsrAction::SendDoViewChange {
view: self.view.get(),
target: primary_candidate,
log_view: self.log_view.get(),
op: current_op,
commit: current_commit,
namespace: self.namespace,
- });
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ actions.push(action);
// If we are the primary candidate, record our own DVC
if primary_candidate == self.replica {
@@ -930,7 +1044,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// Check if we now have quorum
if dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() {
self.do_view_change_quorum.set(true);
- actions.extend(self.complete_view_change_as_primary());
+ actions.extend(self.complete_view_change_as_primary(plane));
}
}
}
@@ -946,7 +1060,11 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// # Panics
/// If `header.namespace` does not match this replica's namespace.
- pub fn handle_do_view_change(&self, header: &DoViewChangeHeader) -> Vec<VsrAction> {
+ pub fn handle_do_view_change(
+ &self,
+ plane: PlaneKind,
+ header: &DoViewChangeHeader,
+ ) -> Vec<VsrAction> {
assert_eq!(
header.namespace, self.namespace,
"DVC routed to wrong group"
@@ -966,6 +1084,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// If DVC is for a higher view, advance to that view
if msg_view > self.view.get() {
+ let old_view = self.view.get();
self.view.set(msg_view);
self.status.set(Status::ViewChange);
self.reset_view_change_state();
@@ -982,11 +1101,29 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
timeouts.start(TimeoutKind::ViewChangeStatus);
}
+ emit_sim_event(
+ SimEventKind::ViewChangeStarted,
+ &ViewChangeLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ old_view,
+ new_view: msg_view,
+ reason: ViewChangeReason::ReceivedDoViewChange,
+ },
+ );
+
// Send our own SVC
- actions.push(VsrAction::SendStartViewChange {
+ let action = VsrAction::SendStartViewChange {
view: msg_view,
namespace: self.namespace,
- });
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ actions.push(action);
}
// Only the primary candidate processes DVCs for quorum
@@ -1032,7 +1169,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
&& dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum()
{
self.do_view_change_quorum.set(true);
- actions.extend(self.complete_view_change_as_primary());
+ actions.extend(self.complete_view_change_as_primary(plane));
}
actions
@@ -1047,7 +1184,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
///
/// # Panics
/// If `header.namespace` does not match this replica's namespace.
- pub fn handle_start_view(&self, header: &StartViewHeader) -> Vec<VsrAction> {
+ pub fn handle_start_view(&self, plane: PlaneKind, header: &StartViewHeader) -> Vec<VsrAction> {
assert_eq!(header.namespace, self.namespace, "SV routed to wrong group");
let from_replica = header.replica;
let msg_view = header.view;
@@ -1094,19 +1231,32 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// Send PrepareOK for uncommitted ops (commit+1 to op)
let mut actions = Vec::new();
for op_num in (msg_commit + 1)..=msg_op {
- actions.push(VsrAction::SendPrepareOk {
+ let action = VsrAction::SendPrepareOk {
view: msg_view,
op: op_num,
target: from_replica,
namespace: self.namespace,
- });
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ actions.push(action);
}
+ emit_replica_event(
+ SimEventKind::ReplicaStateChanged,
+ &ReplicaLogContext::from_consensus(self, plane),
+ );
+
actions
}
/// Complete view change as the new primary after collecting DVC quorum.
- fn complete_view_change_as_primary(&self) -> Vec<VsrAction> {
+ fn complete_view_change_as_primary(&self, plane: PlaneKind) -> Vec<VsrAction> {
let dvc_array = self.do_view_change_from_all_replicas.borrow();
let Some(winner) = dvc_select_winner(&dvc_array) else {
@@ -1140,23 +1290,39 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
timeouts.start(TimeoutKind::CommitMessage);
}
- vec![VsrAction::SendStartView {
+ let state = ReplicaLogContext::from_consensus(self, plane);
+ emit_replica_event(SimEventKind::PrimaryElected, &state);
+ emit_replica_event(SimEventKind::ReplicaStateChanged, &state);
+
+ let action = VsrAction::SendStartView {
view: self.view.get(),
op: new_op,
commit: max_commit,
namespace: self.namespace,
- }]
+ };
+ emit_sim_event(
+ SimEventKind::ControlMessageScheduled,
+ &ControlActionLogEvent::from_vsr_action(
+ ReplicaLogContext::from_consensus(self, plane),
+ &action,
+ ),
+ );
+ vec![action]
}
/// Handle a `PrepareOk` message from a replica.
///
- /// Returns `true` if quorum was just reached for this op.
+ /// Returns rich ack-progress information for structured logging.
/// Caller (`on_ack`) should validate `is_primary` and status before calling.
///
/// # Panics
/// - If `header.command` is not `Command2::PrepareOk`.
/// - If `header.replica >= self.replica_count`.
- pub fn handle_prepare_ok(&self, header: &PrepareOkHeader) -> bool {
+ pub fn handle_prepare_ok(
+ &self,
+ plane: PlaneKind,
+ header: &PrepareOkHeader,
+ ) -> PrepareOkOutcome {
assert_eq!(header.command, Command2::PrepareOk);
assert!(
header.replica < self.replica_count,
@@ -1166,48 +1332,78 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
// Ignore if from older view
if header.view < self.view() {
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::OlderView,
+ };
}
// Ignore if from newer view
if header.view > self.view() {
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::NewerView,
+ };
}
// Ignore if syncing
if self.is_syncing() {
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::Syncing,
+ };
}
// Find the prepare in our pipeline
let mut pipeline = self.pipeline.borrow_mut();
- let Some(entry) = pipeline.message_by_op_mut(header.op) else {
+ let Some(entry) = pipeline.entry_by_op_mut(header.op) else {
// Not in pipeline - could be old/duplicate or already committed
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::UnknownPrepare,
+ };
};
// Verify checksum matches
if entry.header.checksum != header.prepare_checksum {
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::ChecksumMismatch,
+ };
}
// Check for duplicate ack
if entry.has_ack(header.replica) {
- return false;
+ return PrepareOkOutcome::Ignored {
+ reason: IgnoreReason::DuplicateAck,
+ };
}
// Record the ack from this replica
let ack_count = entry.add_ack(header.replica);
let quorum = self.quorum();
+ let quorum_reached = ack_count >= quorum && !entry.ok_quorum_received;
// Check if we've reached quorum
- if ack_count >= quorum && !entry.ok_quorum_received {
+ if quorum_reached {
entry.ok_quorum_received = true;
- return true;
}
- false
+ drop(pipeline);
+
+ emit_sim_event(
+ SimEventKind::PrepareAcked,
+ &AckLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ op: header.op,
+ prepare_checksum: header.prepare_checksum,
+ ack_from_replica: header.replica,
+ ack_count,
+ quorum,
+ quorum_reached,
+ },
+ );
+
+ PrepareOkOutcome::Accepted {
+ ack_count,
+ quorum_reached,
+ }
}
/// Enqueue a self-addressed message for processing in the next loopback drain.
@@ -1257,7 +1453,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
impl<B, P> Project<Message<PrepareHeader>, VsrConsensus<B, P>> for Message<RequestHeader>
where
B: MessageBus,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
type Consensus = VsrConsensus<B, P>;
@@ -1290,7 +1486,7 @@ where
impl<B, P> Project<Message<PrepareOkHeader>, VsrConsensus<B, P>> for Message<PrepareHeader>
where
B: MessageBus,
- P: Pipeline<Message = Self, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
type Consensus = VsrConsensus<B, P>;
@@ -1320,7 +1516,7 @@ where
impl<B, P> Consensus for VsrConsensus<B, P>
where
B: MessageBus,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
type MessageBus = B;
#[rustfmt::skip] // Scuffed formatter. TODO: Make the naming less ambiguous for `Message`.
@@ -1336,11 +1532,28 @@ where
// (push_loopback / drain_loopback_into) rather than inline here,
// so that WAL persistence can happen between pipeline insertion
// and ack recording.
- fn pipeline_message(&self, message: Self::Message<Self::ReplicateHeader>) {
+ fn pipeline_message(&self, plane: PlaneKind, message: &Self::Message<Self::ReplicateHeader>) {
assert!(self.is_primary(), "only primary can pipeline messages");
let mut pipeline = self.pipeline.borrow_mut();
- pipeline.push_message(message);
+ pipeline.push(PipelineEntry::new(*message.header()));
+ let pipeline_depth = pipeline.len();
+ drop(pipeline);
+
+ let header = message.header();
+ emit_sim_event(
+ SimEventKind::PrepareQueued,
+ &PrepareLogEvent {
+ replica: ReplicaLogContext::from_consensus(self, plane),
+ op: header.op,
+ parent_checksum: header.parent,
+ prepare_checksum: header.checksum,
+ client_id: header.client,
+ request_id: header.request,
+ operation: header.operation,
+ pipeline_depth,
+ },
+ );
}
fn verify_pipeline(&self) {
@@ -1357,6 +1570,7 @@ where
}
fn is_syncing(&self) -> bool {
- self.is_syncing()
+ // TODO: for now return false. we have to add syncing related setup to VsrConsensus to make this work.
+ false
}
}
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs
index 31ef7c6dd..200ee87ad 100644
--- a/core/consensus/src/lib.rs
+++ b/core/consensus/src/lib.rs
@@ -24,20 +24,19 @@ pub trait Project<T, C: Consensus> {
}
pub trait Pipeline {
- type Message;
type Entry;
- fn push_message(&mut self, message: Self::Message);
+ fn push(&mut self, entry: Self::Entry);
- fn pop_message(&mut self) -> Option<Self::Entry>;
+ fn pop(&mut self) -> Option<Self::Entry>;
fn clear(&mut self);
- fn message_by_op(&self, op: u64) -> Option<&Self::Entry>;
+ fn entry_by_op(&self, op: u64) -> Option<&Self::Entry>;
- fn message_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry>;
+ fn entry_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry>;
- fn message_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry>;
+ fn entry_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry>;
fn head(&self) -> Option<&Self::Entry>;
@@ -45,6 +44,8 @@ pub trait Pipeline {
fn is_empty(&self) -> bool;
+ fn len(&self) -> usize;
+
fn verify(&self);
}
@@ -62,9 +63,9 @@ pub trait Consensus: Sized {
type AckHeader: ConsensusHeader;
type Sequencer: Sequencer;
- type Pipeline: Pipeline<Message = Self::Message<Self::ReplicateHeader>>;
+ type Pipeline: Pipeline;
- fn pipeline_message(&self, message: Self::Message<Self::ReplicateHeader>);
+ fn pipeline_message(&self, plane: PlaneKind, message: &Self::Message<Self::ReplicateHeader>);
fn verify_pipeline(&self);
fn is_follower(&self) -> bool;
@@ -84,11 +85,11 @@ where
{
fn on_request(&self, message: RequestMessage<C>) -> impl Future<Output = ()>
where
- RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C> + Clone;
+ RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C>;
fn on_replicate(&self, message: ReplicateMessage<C>) -> impl Future<Output = ()>
where
- ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C> + Clone;
+ ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C>;
fn on_ack(&self, message: AckMessage<C>) -> impl Future<Output = ()>;
}
@@ -110,6 +111,8 @@ mod namespaced_pipeline;
pub use namespaced_pipeline::*;
mod plane_helpers;
pub use plane_helpers::*;
+mod observability;
+pub use observability::*;
mod view_change_quorum;
pub use view_change_quorum::*;
diff --git a/core/consensus/src/namespaced_pipeline.rs b/core/consensus/src/namespaced_pipeline.rs
index 7e77d3e16..71e18fb71 100644
--- a/core/consensus/src/namespaced_pipeline.rs
+++ b/core/consensus/src/namespaced_pipeline.rs
@@ -135,7 +135,7 @@ impl NamespacedPipeline {
break;
}
// Still in a queue means not yet drained
- if self.message_by_op(next).is_some() {
+ if self.entry_by_op(next).is_some() {
break;
}
commit = next;
@@ -145,7 +145,6 @@ impl NamespacedPipeline {
}
impl Pipeline for NamespacedPipeline {
- type Message = Message<PrepareHeader>;
type Entry = PipelineEntry;
/// # Panics
@@ -153,13 +152,13 @@ impl Pipeline for NamespacedPipeline {
/// - If ops are not globally sequential.
/// - If the hash chain is broken.
/// - If the namespace is not registered.
- fn push_message(&mut self, message: Self::Message) {
+ fn push(&mut self, entry: Self::Entry) {
assert!(
self.total_count < PIPELINE_PREPARE_QUEUE_MAX,
"namespaced pipeline full"
);
- let header = *message.header();
+ let header = entry.header;
let ns = header.namespace;
if self.total_count > 0 {
@@ -189,13 +188,13 @@ impl Pipeline for NamespacedPipeline {
);
}
- queue.push_back(PipelineEntry::new(header));
+ queue.push_back(entry);
self.total_count += 1;
self.last_push_checksum = header.checksum;
self.last_push_op = header.op;
}
- fn pop_message(&mut self) -> Option<Self::Entry> {
+ fn pop(&mut self) -> Option<Self::Entry> {
let min_ns = self
.queues
.iter()
@@ -219,7 +218,7 @@ impl Pipeline for NamespacedPipeline {
}
/// Linear scan all queues. Ops are globally unique; max 8 entries total.
- fn message_by_op(&self, op: u64) -> Option<&Self::Entry> {
+ fn entry_by_op(&self, op: u64) -> Option<&Self::Entry> {
for queue in self.queues.values() {
for entry in queue {
if entry.header.op == op {
@@ -230,7 +229,7 @@ impl Pipeline for NamespacedPipeline {
None
}
- fn message_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry> {
+ fn entry_by_op_mut(&mut self, op: u64) -> Option<&mut Self::Entry> {
for queue in self.queues.values_mut() {
for entry in queue.iter_mut() {
if entry.header.op == op {
@@ -241,8 +240,8 @@ impl Pipeline for NamespacedPipeline {
None
}
- fn message_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry> {
- let entry = self.message_by_op(op)?;
+ fn entry_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&Self::Entry> {
+ let entry = self.entry_by_op(op)?;
if entry.header.checksum == checksum {
Some(entry)
} else {
@@ -265,6 +264,10 @@ impl Pipeline for NamespacedPipeline {
self.total_count == 0
}
+ fn len(&self) -> usize {
+ self.total_count
+ }
+
fn verify(&self) {
assert!(self.total_count <= PIPELINE_PREPARE_QUEUE_MAX);
@@ -309,10 +312,35 @@ impl Pipeline for NamespacedPipeline {
}
}
+impl NamespacedPipeline {
+ #[allow(clippy::needless_pass_by_value)]
+ pub fn push_message(&mut self, message: Message<PrepareHeader>) {
+ self.push(PipelineEntry::new(*message.header()));
+ }
+
+ pub fn pop_message(&mut self) -> Option<PipelineEntry> {
+ self.pop()
+ }
+
+ #[must_use]
+ pub fn message_by_op(&self, op: u64) -> Option<&PipelineEntry> {
+ self.entry_by_op(op)
+ }
+
+ pub fn message_by_op_mut(&mut self, op: u64) -> Option<&mut PipelineEntry> {
+ self.entry_by_op_mut(op)
+ }
+
+ #[must_use]
+ pub fn message_by_op_and_checksum(&self, op: u64, checksum: u128) -> Option<&PipelineEntry> {
+ self.entry_by_op_and_checksum(op, checksum)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
- use iggy_binary_protocol::Command2;
+ use iggy_binary_protocol::{Command2, Message, PrepareHeader};
fn make_prepare(
op: u64,
diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs
new file mode 100644
index 000000000..865935faf
--- /dev/null
+++ b/core/consensus/src/observability.rs
@@ -0,0 +1,604 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::{Pipeline, PipelineEntry, Status, VsrAction, VsrConsensus};
+use iggy_binary_protocol::Operation;
+use iggy_common::sharding::IggyNamespace;
+use message_bus::MessageBus;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PlaneKind {
+ Metadata,
+ Partitions,
+}
+
+impl PlaneKind {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Metadata => "metadata",
+ Self::Partitions => "partitions",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReplicaRole {
+ Primary,
+ Backup,
+}
+
+impl ReplicaRole {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Primary => "primary",
+ Self::Backup => "backup",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ViewChangeReason {
+ NormalHeartbeatTimeout,
+ ViewChangeStatusTimeout,
+ ReceivedStartViewChange,
+ ReceivedDoViewChange,
+}
+
+impl ViewChangeReason {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::NormalHeartbeatTimeout => "normal_heartbeat_timeout",
+ Self::ViewChangeStatusTimeout => "view_change_status_timeout",
+ Self::ReceivedStartViewChange => "received_start_view_change",
+ Self::ReceivedDoViewChange => "received_do_view_change",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum IgnoreReason {
+ NotPrimary,
+ NotNormal,
+ Syncing,
+ NewerView,
+ OlderView,
+ OldPrepare,
+ UnknownPrepare,
+ DuplicateAck,
+ ChecksumMismatch,
+ InvalidOperation,
+ ApplyFailed,
+ PersistFailed,
+}
+
+impl IgnoreReason {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::NotPrimary => "not_primary",
+ Self::NotNormal => "not_normal",
+ Self::Syncing => "syncing",
+ Self::NewerView => "newer_view",
+ Self::OlderView => "older_view",
+ Self::OldPrepare => "old_prepare",
+ Self::UnknownPrepare => "unknown_prepare",
+ Self::DuplicateAck => "duplicate_ack",
+ Self::ChecksumMismatch => "checksum_mismatch",
+ Self::InvalidOperation => "invalid_operation",
+ Self::ApplyFailed => "apply_failed",
+ Self::PersistFailed => "persist_failed",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ControlActionKind {
+ SendStartViewChange,
+ SendDoViewChange,
+ SendStartView,
+ SendPrepareOk,
+}
+
+impl ControlActionKind {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::SendStartViewChange => "send_start_view_change",
+ Self::SendDoViewChange => "send_do_view_change",
+ Self::SendStartView => "send_start_view",
+ Self::SendPrepareOk => "send_prepare_ok",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SimEventKind {
+ ClientRequestReceived,
+ PrepareQueued,
+ PrepareAcked,
+ OperationCommitted,
+ ClientReplyEmitted,
+ ViewChangeStarted,
+ PrimaryElected,
+ ReplicaStateChanged,
+ NamespaceProgressUpdated,
+ ControlMessageScheduled,
+}
+
+impl SimEventKind {
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::ClientRequestReceived => "ClientRequestReceived",
+ Self::PrepareQueued => "PrepareQueued",
+ Self::PrepareAcked => "PrepareAcked",
+ Self::OperationCommitted => "OperationCommitted",
+ Self::ClientReplyEmitted => "ClientReplyEmitted",
+ Self::ViewChangeStarted => "ViewChangeStarted",
+ Self::PrimaryElected => "PrimaryElected",
+ Self::ReplicaStateChanged => "ReplicaStateChanged",
+ Self::NamespaceProgressUpdated => "NamespaceProgressUpdated",
+ Self::ControlMessageScheduled => "ControlMessageScheduled",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct NamespaceLogContext {
+ pub raw: u64,
+ pub stream_id: Option<u32>,
+ pub topic_id: Option<u32>,
+ pub partition_id: Option<u32>,
+}
+
+impl NamespaceLogContext {
+ #[must_use]
+ pub fn from_raw(plane: PlaneKind, raw: u64) -> Self {
+ if matches!(plane, PlaneKind::Metadata) {
+ return Self {
+ raw,
+ stream_id: None,
+ topic_id: None,
+ partition_id: None,
+ };
+ }
+
+ let namespace = IggyNamespace::from_raw(raw);
+ #[allow(clippy::cast_possible_truncation)]
+ let stream_id = namespace.stream_id() as u32;
+ #[allow(clippy::cast_possible_truncation)]
+ let topic_id = namespace.topic_id() as u32;
+ #[allow(clippy::cast_possible_truncation)]
+ let partition_id = namespace.partition_id() as u32;
+
+ Self {
+ raw,
+ stream_id: Some(stream_id),
+ topic_id: Some(topic_id),
+ partition_id: Some(partition_id),
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ReplicaLogContext {
+ pub plane: PlaneKind,
+ pub cluster_id: u128,
+ pub replica_id: u8,
+ pub namespace: NamespaceLogContext,
+ pub view: u32,
+ pub log_view: u32,
+ pub commit: u64,
+ pub status: Status,
+ pub role: ReplicaRole,
+}
+
+impl ReplicaLogContext {
+ #[must_use]
+ pub fn from_consensus<B, P>(consensus: &VsrConsensus<B, P>, plane: PlaneKind) -> Self
+ where
+ B: MessageBus,
+ P: Pipeline<Entry = PipelineEntry>,
+ {
+ let role = if consensus.is_primary() {
+ ReplicaRole::Primary
+ } else {
+ ReplicaRole::Backup
+ };
+
+ Self {
+ plane,
+ cluster_id: consensus.cluster(),
+ replica_id: consensus.replica(),
+ namespace: NamespaceLogContext::from_raw(plane, consensus.namespace()),
+ view: consensus.view(),
+ log_view: consensus.log_view(),
+ commit: consensus.commit(),
+ status: consensus.status(),
+ role,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct RequestLogEvent {
+ pub replica: ReplicaLogContext,
+ pub client_id: u128,
+ pub request_id: u64,
+ pub operation: Operation,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct PrepareLogEvent {
+ pub replica: ReplicaLogContext,
+ pub op: u64,
+ pub parent_checksum: u128,
+ pub prepare_checksum: u128,
+ pub client_id: u128,
+ pub request_id: u64,
+ pub operation: Operation,
+ pub pipeline_depth: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AckLogEvent {
+ pub replica: ReplicaLogContext,
+ pub op: u64,
+ pub prepare_checksum: u128,
+ pub ack_from_replica: u8,
+ pub ack_count: usize,
+ pub quorum: usize,
+ pub quorum_reached: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CommitLogEvent {
+ pub replica: ReplicaLogContext,
+ pub op: u64,
+ pub client_id: u128,
+ pub request_id: u64,
+ pub operation: Operation,
+ pub pipeline_depth: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ViewChangeLogEvent {
+ pub replica: ReplicaLogContext,
+ pub old_view: u32,
+ pub new_view: u32,
+ pub reason: ViewChangeReason,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ControlActionLogEvent {
+ pub replica: ReplicaLogContext,
+ pub action: ControlActionKind,
+ pub target_replica: Option<u8>,
+ pub op: Option<u64>,
+ pub commit: Option<u64>,
+}
+
+impl ControlActionLogEvent {
+ #[must_use]
+ pub const fn from_vsr_action(replica: ReplicaLogContext, action: &VsrAction) -> Self {
+ match *action {
+ VsrAction::SendStartViewChange { .. } => Self {
+ replica,
+ action: ControlActionKind::SendStartViewChange,
+ target_replica: None,
+ op: None,
+ commit: None,
+ },
+ VsrAction::SendDoViewChange {
+ target, op, commit, ..
+ } => Self {
+ replica,
+ action: ControlActionKind::SendDoViewChange,
+ target_replica: Some(target),
+ op: Some(op),
+ commit: Some(commit),
+ },
+ VsrAction::SendStartView { op, commit, .. } => Self {
+ replica,
+ action: ControlActionKind::SendStartView,
+ target_replica: None,
+ op: Some(op),
+ commit: Some(commit),
+ },
+ VsrAction::SendPrepareOk { target, op, .. } => Self {
+ replica,
+ action: ControlActionKind::SendPrepareOk,
+ target_replica: Some(target),
+ op: Some(op),
+ commit: None,
+ },
+ }
+ }
+}
+
+pub trait StructuredSimEvent {
+ fn emit(&self, sim_event: SimEventKind);
+}
+
+#[must_use]
+pub const fn status_as_str(status: Status) -> &'static str {
+ match status {
+ Status::Normal => "normal",
+ Status::ViewChange => "view_change",
+ Status::Recovering => "recovering",
+ }
+}
+
+#[must_use]
+pub const fn operation_as_str(operation: Operation) -> &'static str {
+ match operation {
+ Operation::Reserved => "reserved",
+ Operation::CreateStream => "create_stream",
+ Operation::UpdateStream => "update_stream",
+ Operation::DeleteStream => "delete_stream",
+ Operation::PurgeStream => "purge_stream",
+ Operation::CreateTopic => "create_topic",
+ Operation::UpdateTopic => "update_topic",
+ Operation::DeleteTopic => "delete_topic",
+ Operation::PurgeTopic => "purge_topic",
+ Operation::CreatePartitions => "create_partitions",
+ Operation::DeletePartitions => "delete_partitions",
+ Operation::DeleteSegments => "delete_segments",
+ Operation::CreateConsumerGroup => "create_consumer_group",
+ Operation::DeleteConsumerGroup => "delete_consumer_group",
+ Operation::CreateUser => "create_user",
+ Operation::UpdateUser => "update_user",
+ Operation::DeleteUser => "delete_user",
+ Operation::ChangePassword => "change_password",
+ Operation::UpdatePermissions => "update_permissions",
+ Operation::CreatePersonalAccessToken => "create_personal_access_token",
+ Operation::DeletePersonalAccessToken => "delete_personal_access_token",
+ Operation::SendMessages => "send_messages",
+ Operation::StoreConsumerOffset => "store_consumer_offset",
+ }
+}
+
+#[must_use]
+pub const fn namespace_component(component: Option<u32>) -> u32 {
+ match component {
+ Some(value) => value,
+ None => 0,
+ }
+}
+
+pub fn emit_sim_event<T>(sim_event: SimEventKind, event: &T)
+where
+ T: StructuredSimEvent,
+{
+ event.emit(sim_event);
+}
+
+pub fn emit_replica_event(sim_event: SimEventKind, ctx: &ReplicaLogContext) {
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ );
+}
+
+pub fn emit_namespace_progress_event(
+ sim_event: SimEventKind,
+ ctx: &ReplicaLogContext,
+ op: u64,
+ pipeline_depth: usize,
+) {
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ op,
+ pipeline_depth,
+ );
+}
+
+impl StructuredSimEvent for RequestLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ client_id = self.client_id,
+ request_id = self.request_id,
+ operation = operation_as_str(self.operation),
+ );
+ }
+}
+
+impl StructuredSimEvent for PrepareLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ op = self.op,
+ parent_checksum = self.parent_checksum,
+ prepare_checksum = self.prepare_checksum,
+ client_id = self.client_id,
+ request_id = self.request_id,
+ operation = operation_as_str(self.operation),
+ pipeline_depth = self.pipeline_depth,
+ );
+ }
+}
+
+impl StructuredSimEvent for AckLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ op = self.op,
+ prepare_checksum = self.prepare_checksum,
+ ack_from_replica = self.ack_from_replica,
+ ack_count = self.ack_count,
+ quorum = self.quorum,
+ quorum_reached = self.quorum_reached,
+ );
+ }
+}
+
+impl StructuredSimEvent for CommitLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ op = self.op,
+ client_id = self.client_id,
+ request_id = self.request_id,
+ operation = operation_as_str(self.operation),
+ pipeline_depth = self.pipeline_depth,
+ );
+ }
+}
+
+impl StructuredSimEvent for ViewChangeLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ old_view = self.old_view,
+ new_view = self.new_view,
+ reason = self.reason.as_str(),
+ );
+ }
+}
+
+impl StructuredSimEvent for ControlActionLogEvent {
+ fn emit(&self, sim_event: SimEventKind) {
+ let ctx = self.replica;
+ tracing::event!(
+ target: "iggy.sim",
+ tracing::Level::DEBUG,
+ sim_event = sim_event.as_str(),
+ plane = ctx.plane.as_str(),
+ cluster_id = ctx.cluster_id,
+ replica_id = ctx.replica_id,
+ namespace_raw = ctx.namespace.raw,
+ stream_id = namespace_component(ctx.namespace.stream_id),
+ topic_id = namespace_component(ctx.namespace.topic_id),
+ partition_id = namespace_component(ctx.namespace.partition_id),
+ view = ctx.view,
+ log_view = ctx.log_view,
+ commit = ctx.commit,
+ status = status_as_str(ctx.status),
+ role = ctx.role.as_str(),
+ action = self.action.as_str(),
+ target_replica = self.target_replica.unwrap_or_default(),
+ op = self.op.unwrap_or_default(),
+ action_commit = self.commit.unwrap_or_default(),
+ );
+ }
+}
diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs
index fdfc42c31..443682f8f 100644
--- a/core/consensus/src/plane_helpers.rs
+++ b/core/consensus/src/plane_helpers.rs
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{Consensus, Pipeline, PipelineEntry, Sequencer, Status, VsrConsensus};
+use crate::{
+ Consensus, IgnoreReason, Pipeline, PipelineEntry, PlaneKind, PrepareOkOutcome, Sequencer,
+ Status, VsrConsensus,
+};
use iggy_binary_protocol::{
Command2, GenericHeader, Message, PrepareHeader, PrepareOkHeader, ReplyHeader,
};
+use iobuf::Owned;
use message_bus::MessageBus;
use std::ops::AsyncFnOnce;
@@ -33,11 +37,11 @@ use std::ops::AsyncFnOnce;
#[allow(clippy::future_not_send)]
pub async fn pipeline_prepare_common<C, F>(
consensus: &C,
+ plane: PlaneKind,
prepare: C::Message<C::ReplicateHeader>,
on_replicate: F,
) where
C: Consensus,
- C::Message<C::ReplicateHeader>: Clone,
F: AsyncFnOnce(C::Message<C::ReplicateHeader>) -> (),
{
assert!(!consensus.is_follower(), "on_request: primary only");
@@ -45,8 +49,8 @@ pub async fn pipeline_prepare_common<C, F>(
assert!(!consensus.is_syncing(), "on_request: must not be syncing");
consensus.verify_pipeline();
- consensus.pipeline_message(prepare.clone());
- on_replicate(prepare.clone()).await;
+ consensus.pipeline_message(plane, &prepare);
+ on_replicate(prepare).await;
}
/// Shared commit-based old-prepare fence.
@@ -57,7 +61,7 @@ pub const fn fence_old_prepare_by_commit<B, P>(
) -> bool
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
header.op <= consensus.commit()
}
@@ -73,11 +77,12 @@ where
pub async fn replicate_to_next_in_chain<B, P>(
consensus: &VsrConsensus<B, P>,
message: Message<PrepareHeader>,
-) where
+) -> Message<PrepareHeader>
+where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
- let header = message.header();
+ let header = *message.header();
assert_eq!(header.command, Command2::Prepare);
assert!(header.op > consensus.commit());
@@ -86,17 +91,20 @@ pub async fn replicate_to_next_in_chain<B, P>(
let primary = consensus.primary_index(header.view);
if next == primary {
- return;
+ return message;
}
assert_ne!(next, consensus.replica());
// TODO: Propagate send error instead of panicking; requires bus error design.
- consensus
+ let returned = consensus
.message_bus()
.send_to_replica(next, message.into_generic())
.await
.unwrap();
+ returned
+ .try_into_typed()
+ .expect("replica send must return the same prepare message")
}
/// Shared preflight checks for `on_replicate`.
@@ -112,25 +120,25 @@ pub async fn replicate_to_next_in_chain<B, P>(
pub fn replicate_preflight<B, P>(
consensus: &VsrConsensus<B, P>,
header: &PrepareHeader,
-) -> Result<u64, &'static str>
+) -> Result<u64, IgnoreReason>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
assert_eq!(header.command, Command2::Prepare);
if consensus.is_syncing() {
- return Err("sync");
+ return Err(IgnoreReason::Syncing);
}
let current_op = consensus.sequencer().current_sequence();
if consensus.status() != Status::Normal {
- return Err("not normal state");
+ return Err(IgnoreReason::NotNormal);
}
if header.view > consensus.view() {
- return Err("newer view");
+ return Err(IgnoreReason::NewerView);
}
if consensus.is_follower() {
@@ -145,17 +153,17 @@ where
/// # Errors
/// Returns a static error string if the replica is not primary or not in
/// normal status.
-pub fn ack_preflight<B, P>(consensus: &VsrConsensus<B, P>) -> Result<(), &'static str>
+pub fn ack_preflight<B, P>(consensus: &VsrConsensus<B, P>) -> Result<(), IgnoreReason>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
if !consensus.is_primary() {
- return Err("not primary");
+ return Err(IgnoreReason::NotPrimary);
}
if consensus.status() != Status::Normal {
- return Err("not normal");
+ return Err(IgnoreReason::NotNormal);
}
Ok(())
@@ -166,18 +174,28 @@ where
/// After recording the ack, walks forward from `current_commit + 1` advancing
/// the commit number only while consecutive ops have achieved quorum. This
/// prevents committing ops that have gaps in quorum acknowledgment.
-pub fn ack_quorum_reached<B, P>(consensus: &VsrConsensus<B, P>, ack: &PrepareOkHeader) -> bool
+pub fn ack_quorum_reached<B, P>(
+ consensus: &VsrConsensus<B, P>,
+ plane: PlaneKind,
+ ack: &PrepareOkHeader,
+) -> bool
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
- if !consensus.handle_prepare_ok(ack) {
+ if !matches!(
+ consensus.handle_prepare_ok(plane, ack),
+ PrepareOkOutcome::Accepted {
+ quorum_reached: true,
+ ..
+ }
+ ) {
return false;
}
let pipeline = consensus.pipeline().borrow();
let mut new_commit = consensus.commit();
- while let Some(entry) = pipeline.message_by_op(new_commit + 1) {
+ while let Some(entry) = pipeline.entry_by_op(new_commit + 1) {
if !entry.ok_quorum_received {
break;
}
@@ -199,11 +217,11 @@ where
/// by the current commit frontier.
///
/// # Panics
-/// If `head()` returns `Some` but `pop_message()` returns `None` (unreachable).
+/// If `head()` returns `Some` but `pop()` returns `None` (unreachable).
pub fn drain_committable_prefix<B, P>(consensus: &VsrConsensus<B, P>) -> Vec<PipelineEntry>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
let commit = consensus.commit();
let mut drained = Vec::new();
@@ -215,7 +233,7 @@ where
}
let entry = pipeline
- .pop_message()
+ .pop()
.expect("drain_committable_prefix: head exists");
drained.push(entry);
}
@@ -235,7 +253,7 @@ pub fn build_reply_message<B, P>(
) -> Message<ReplyHeader>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
let header_size = std::mem::size_of::<ReplyHeader>();
let total_size = header_size + body.len();
@@ -268,8 +286,10 @@ where
buffer[header_size..].copy_from_slice(&body);
}
- Message::<ReplyHeader>::from_bytes(buffer.freeze())
- .expect("build_reply_message: constructed header must be valid")
+ // TODO: Remove this copy once replies stop round-tripping through `Bytes`
+ // and the binary protocol uses `Owned` end-to-end.
+ Message::try_from(Owned::<4096>::copy_from_slice(buffer.as_ref()))
+ .expect("reply buffer must contain a valid reply message")
}
/// Verify hash chain would not break if we add this header.
@@ -301,7 +321,7 @@ pub async fn send_prepare_ok<B, P>(
is_persisted: Option<bool>,
) where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
{
assert_eq!(header.command, Command2::Prepare);
@@ -389,17 +409,17 @@ mod tests {
async fn send_to_client(
&self,
_client_id: Self::Client,
- _data: Self::Data,
- ) -> Result<(), IggyError> {
- Ok(())
+ data: Self::Data,
+ ) -> Result<Self::Data, IggyError> {
+ Ok(data)
}
async fn send_to_replica(
&self,
_replica: Self::Replica,
- _data: Self::Data,
- ) -> Result<(), IggyError> {
- Ok(())
+ data: Self::Data,
+ ) -> Result<Self::Data, IggyError> {
+ Ok(data)
}
}
@@ -500,7 +520,7 @@ mod tests {
namespace: 0,
reserved: [0; 120],
};
- let _ = consensus.handle_start_view_change(&svc);
+ let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc);
// Simulate an in-flight loopback message queued between SVC and DVC quorum.
let stale_msg = Message::<PrepareOkHeader>::new(std::mem::size_of::<PrepareOkHeader>());
@@ -523,7 +543,7 @@ mod tests {
log_view: 0,
reserved: [0; 100],
};
- let actions = consensus.handle_do_view_change(&dvc);
+ let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc);
// View change completed: should have SendStartView action.
assert!(
@@ -597,17 +617,19 @@ mod tests {
async fn send_to_client(
&self,
_client_id: Self::Client,
- _data: Self::Data,
- ) -> Result<(), IggyError> {
- Ok(())
+ data: Self::Data,
+ ) -> Result<Self::Data, IggyError> {
+ Ok(data)
}
async fn send_to_replica(
&self,
replica: Self::Replica,
data: Self::Data,
- ) -> Result<(), IggyError> {
- self.sent.borrow_mut().push((replica, data));
- Ok(())
+ ) -> Result<Self::Data, IggyError> {
+ let stored = data.deep_copy();
+ let returned = data.deep_copy();
+ self.sent.borrow_mut().push((replica, stored));
+ Ok(returned)
}
}
@@ -647,9 +669,9 @@ mod tests {
let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new());
consensus.init();
- consensus.pipeline_message(prepare_message(1, 0, 10));
- consensus.pipeline_message(prepare_message(2, 10, 20));
- consensus.pipeline_message(prepare_message(3, 20, 30));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(1, 0, 10));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(2, 10, 20));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(3, 20, 30));
consensus.advance_commit_number(3);
@@ -664,9 +686,9 @@ mod tests {
let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new());
consensus.init();
- consensus.pipeline_message(prepare_message(5, 0, 50));
- consensus.pipeline_message(prepare_message(6, 50, 60));
- consensus.pipeline_message(prepare_message(7, 60, 70));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(5, 0, 50));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(6, 50, 60));
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare_message(7, 60, 70));
consensus.advance_commit_number(6);
let drained = drain_committable_prefix(&consensus);
diff --git a/core/consensus/src/plane_mux.rs b/core/consensus/src/plane_mux.rs
index 6ad5078ff..c1e09b790 100644
--- a/core/consensus/src/plane_mux.rs
+++ b/core/consensus/src/plane_mux.rs
@@ -50,14 +50,14 @@ where
{
async fn on_request(&self, message: RequestMessage<C>)
where
- RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C> + Clone,
+ RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C>,
{
self.inner.on_request(message).await;
}
async fn on_replicate(&self, message: ReplicateMessage<C>)
where
- ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C> + Clone,
+ ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C>,
{
self.inner.on_replicate(message).await;
}
@@ -75,13 +75,13 @@ where
{
async fn on_request(&self, _message: RequestMessage<C>)
where
- RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C> + Clone,
+ RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C>,
{
}
async fn on_replicate(&self, _message: ReplicateMessage<C>)
where
- ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C> + Clone,
+ ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C>,
{
}
@@ -150,7 +150,7 @@ where
{
async fn on_request(&self, message: RequestMessage<C>)
where
- RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C> + Clone,
+ RequestMessage<C>: Project<ReplicateMessage<C>, C, Consensus = C>,
{
if self.0.is_applicable(&message) {
self.0.on_request(message).await;
@@ -161,7 +161,7 @@ where
async fn on_replicate(&self, message: ReplicateMessage<C>)
where
- ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C> + Clone,
+ ReplicateMessage<C>: Project<AckMessage<C>, C, Consensus = C>,
{
if self.0.is_applicable(&message) {
self.0.on_replicate(message).await;
diff --git a/core/iobuf/Cargo.toml b/core/iobuf/Cargo.toml
index 62a8b8cc1..9ec4e8cd6 100644
--- a/core/iobuf/Cargo.toml
+++ b/core/iobuf/Cargo.toml
@@ -23,3 +23,4 @@ license = "Apache-2.0"
[dependencies]
aligned-vec = "0.6"
+compio-buf = "0.8.0"
diff --git a/core/iobuf/src/lib.rs b/core/iobuf/src/lib.rs
index 4b9006672..0e57c7b79 100644
--- a/core/iobuf/src/lib.rs
+++ b/core/iobuf/src/lib.rs
@@ -15,12 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use std::mem::ManuallyDrop;
+use std::ops::{Deref, DerefMut, RangeBounds};
use std::ptr::NonNull;
use std::slice;
use std::sync::atomic::{AtomicUsize, Ordering, fence};
use aligned_vec::{AVec, ConstAlign};
+use compio_buf::IoBuf;
#[derive(Debug)]
pub struct Owned<const ALIGN: usize = 4096> {
@@ -39,162 +40,140 @@ impl<const ALIGN: usize> From<Owned<ALIGN>> for AVec<u8, ConstAlign<ALIGN>> {
}
}
-impl<const ALIGN: usize> Owned<ALIGN> {
+pub struct Prefix<const ALIGN: usize> {
+ inner: Extent<ALIGN>,
+}
+
+unsafe impl<const ALIGN: usize> Send for Prefix<ALIGN> {}
+
+impl<const ALIGN: usize> Prefix<ALIGN> {
+ pub fn from_aligned_vec(vec: AVec<u8, ConstAlign<ALIGN>>) -> Self {
+ Self {
+ inner: extent_from_aligned_vec(vec),
+ }
+ }
+
pub fn as_slice(&self) -> &[u8] {
- &self.inner
+ self.inner.as_slice()
}
pub fn as_mut_slice(&mut self) -> &mut [u8] {
- &mut self.inner
+ unsafe { self.inner.as_mut_slice() }
}
- /// Split `Owned` buffer into two halves
- ///
- /// # Panics
- /// Panics if `split_at > self.len()`.
- pub fn split_at(self, split_at: usize) -> TwoHalves<ALIGN> {
- assert!(split_at <= self.inner.len());
+ pub fn freeze(self) -> Frozen<ALIGN> {
+ freeze_extent(self.inner)
+ }
- // Take ownership of the AVec's allocation. After this, we are responsible
- // for deallocating via `AVec::from_raw_parts` or equivalent.
- let (ptr, _, len, capacity) = self.inner.into_raw_parts();
+ pub fn aliases(&self, tail: &Frozen<ALIGN>) -> bool {
+ std::ptr::addr_eq(self.inner.ctrlb.as_ptr(), tail.inner.ctrlb.as_ptr())
+ }
- // SAFETY: both pointers are constructed from the same `Inner` allocation, the split_at bounds are validated.
- // The control block captures original `Inner` metadata to allow reconstructing the original frame for merging/dropping.
- // The ptr provenance rules are maintained by the use of `NonNull` apis.
- let base: NonNull<u8> = unsafe { NonNull::new_unchecked(ptr) };
- let tail = unsafe { NonNull::new_unchecked(ptr.add(split_at)) };
- let ctrlb = ControlBlock::new(base, len, capacity);
- // We need to increment the ref_count as the resulting halves will both point to the same control block.
- unsafe { ctrlb.as_ref().ref_count.fetch_add(1, Ordering::Relaxed) };
+ pub fn can_coalesce_with(&self, tail: &Frozen<ALIGN>) -> bool {
+ can_coalesce_prefix_with_tail(&self.inner, tail)
+ }
- TwoHalves {
- inner: (
- Extent {
- ptr: base,
- len: split_at,
- ctrlb,
- _pad: 0,
- },
- Extent {
- ptr: tail,
- len: len - split_at,
- ctrlb,
- _pad: 0,
- },
- ),
+ pub fn try_coalesce_with(
+ self,
+ tail: Frozen<ALIGN>,
+ ) -> Result<Frozen<ALIGN>, (Prefix<ALIGN>, Frozen<ALIGN>)> {
+ match try_coalesce_prefix_with_tail(self.inner, tail) {
+ Ok(merged) => Ok(merged),
+ Err((prefix, tail)) => Err((Prefix { inner: prefix }, tail)),
}
}
}
-pub struct TwoHalves<const ALIGN: usize> {
- inner: (Extent<ALIGN>, Extent<ALIGN>),
-}
-
-// SAFETY: `TwoHalves` can be sent across threads as long as the caller ensures that the head half is not shared between
-// threads (e.g. by cloning, which creates a new head half),
-// and that the tail half is only shared immutably (e.g. by cloning, which shares the tail half immutably).
-unsafe impl<const ALIGN: usize> Send for TwoHalves<ALIGN> {}
-
-impl<const ALIGN: usize> TwoHalves<ALIGN> {
- pub fn head(&self) -> &[u8] {
- self.inner.0.as_slice()
+impl<const ALIGN: usize> Owned<ALIGN> {
+ pub fn zeroed(len: usize) -> Self {
+ let mut inner: AVec<u8, ConstAlign<ALIGN>> = AVec::new(ALIGN);
+ inner.resize(len, 0);
+ Self { inner }
}
- pub fn head_mut(&mut self) -> &mut [u8] {
- // SAFETY: We are accessing the head half mutably, this is the only correct operation, as the head is not shared between clones,
- // instead it gets copied.
- unsafe { self.inner.0.as_mut_slice() }
+ pub fn copy_from_slice(data: &[u8]) -> Self {
+ let mut inner: AVec<u8, ConstAlign<ALIGN>> = AVec::new(ALIGN);
+ inner.extend_from_slice(data);
+ Self { inner }
}
- pub fn tail(&self) -> &[u8] {
- self.inner.1.as_slice()
+ pub fn as_slice(&self) -> &[u8] {
+ &self.inner
}
- pub fn split_point(&self) -> usize {
- self.inner.0.len
+ pub fn as_mut_slice(&mut self) -> &mut [u8] {
+ &mut self.inner
}
- pub fn total_len(&self) -> usize {
- self.inner.0.len + self.inner.1.len
- }
+ pub fn split_at(self, split_at: usize) -> (Prefix<ALIGN>, Frozen<ALIGN>) {
+ assert!(split_at <= self.inner.len());
- pub fn try_merge(self) -> Result<Owned<ALIGN>, Self> {
- let ctrlb_eq = std::ptr::addr_eq(self.inner.0.ctrlb.as_ptr(), self.inner.1.ctrlb.as_ptr());
- // SAFETY: `inner.1.ctrlb` points to a live control block while `self` is alive.
- let ref_count = unsafe {
- self.inner
- .1
- .ctrlb
- .as_ref()
- .ref_count
- .load(Ordering::Acquire)
+ let (ptr, _, len, capacity) = self.inner.into_raw_parts();
+ let base: NonNull<u8> = unsafe { NonNull::new_unchecked(ptr) };
+ let tail = unsafe { NonNull::new_unchecked(ptr.add(split_at)) };
+ let ctrlb = ControlBlock::new(base, len, capacity);
+ unsafe { ctrlb.as_ref().ref_count.fetch_add(1, Ordering::Relaxed) };
+
+ let prefix = Prefix {
+ inner: Extent {
+ ptr: base,
+ len: split_at,
+ ctrlb,
+ },
};
- // When ctrlb_eq, both extents share the same control block with refcount 2.
- // When !ctrlb_eq (after clone), the tail has its own refcount.
- let is_unique = if ctrlb_eq {
- ref_count == 2
- } else {
- ref_count == 1
+ let tail = Frozen {
+ inner: Extent {
+ ptr: tail,
+ len: len - split_at,
+ ctrlb,
+ },
};
- if !is_unique {
- return Err(self);
+ (prefix, tail)
+ }
+}
+
+impl<const ALIGN: usize> Prefix<ALIGN> {
+ pub fn copy_from_slice(src: &[u8]) -> Self {
+ Self {
+ inner: Extent::copy_from_slice(src),
}
+ }
- // Transfer ownership to prevent Extent::drop from running.
- // SAFETY: We read the inner tuple out of ManuallyDrop, which won't run the compiler-generated drop.
- let this = ManuallyDrop::new(self);
- let (head, tail) = unsafe { std::ptr::read(&this.inner) };
- let split_at = head.len;
+ pub fn len(&self) -> usize {
+ self.inner.len
+ }
- // SAFETY: `tail.ctrlb` is unique at this point (is_unique checked above),
- // If `head.ctrlb != tail.ctrlb`, the head owns a standalone allocation
- // that must be released after copying.
- unsafe {
- if !ctrlb_eq {
- let dst_ctrlb = tail.ctrlb.as_ref();
-
- // We are patching up the original allocation, with the current head data, so that the resulting `Owned` has correct content.
- let dst = slice::from_raw_parts_mut(dst_ctrlb.base.as_ptr(), split_at);
- dst.copy_from_slice(head.as_slice());
- }
-
- // Dropping the head in `ctrlb_eq` case, should decrease the refcount to 1, so it's safe to reuse tail control block.
- // In case when head was it's own allocation, we guarantee that it's always unique.
- drop(head);
- let tail_ctrlb = tail.ctrlb;
- // Prevent tail Drop from running, we're taking ownership of the control block.
- std::mem::forget(tail);
-
- let ctrlb = reclaim_unique_control_block(tail_ctrlb);
- // SAFETY: `ctrlb.base,capacity` were captured from an `AVec<u8>` allocation and
- // are now exclusively owned by this path.
- let inner = AVec::from_raw_parts(ctrlb.base.as_ptr(), ALIGN, ctrlb.len, ctrlb.capacity);
- Ok(Owned { inner })
- }
+ pub fn is_empty(&self) -> bool {
+ self.inner.len == 0
}
}
-impl<const ALIGN: usize> Clone for TwoHalves<ALIGN> {
+impl<const ALIGN: usize> Clone for Prefix<ALIGN> {
fn clone(&self) -> Self {
Self {
- inner: (
- Extent::<ALIGN>::copy_from_slice(self.head()),
- self.inner.1.clone(),
- ),
+ inner: Extent::copy_from_slice(self.inner.as_slice()),
}
}
}
-impl<const ALIGN: usize> std::fmt::Debug for TwoHalves<ALIGN> {
+impl<const ALIGN: usize> std::fmt::Debug for Prefix<ALIGN> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("TwoHalves")
- .field("split_point", &self.split_point())
- .field("head_len", &self.inner.0.len)
- .field("tail_len", &self.inner.1.len)
- .field("halves_alias", &(self.inner.0.ctrlb == self.inner.1.ctrlb))
- .finish()
+ f.debug_struct("Prefix").field("len", &self.len()).finish()
+ }
+}
+
+impl<const ALIGN: usize> Deref for Prefix<ALIGN> {
+ type Target = [u8];
+
+ fn deref(&self) -> &Self::Target {
+ self.inner.as_slice()
+ }
+}
+
+impl<const ALIGN: usize> DerefMut for Prefix<ALIGN> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ self.as_mut_slice()
}
}
@@ -203,12 +182,19 @@ pub struct Frozen<const ALIGN: usize> {
inner: Extent<ALIGN>,
}
+unsafe impl<const ALIGN: usize> Send for Frozen<ALIGN> {}
+
+impl<const ALIGN: usize> From<Extent<ALIGN>> for Frozen<ALIGN> {
+ fn from(inner: Extent<ALIGN>) -> Self {
+ Self { inner }
+ }
+}
+
impl<const ALIGN: usize> From<Owned<ALIGN>> for Frozen<ALIGN> {
fn from(value: Owned<ALIGN>) -> Self {
let inner = value.inner;
let (ptr, _, len, capacity) = inner.into_raw_parts();
- // SAFETY: The `Owned` buffer is guaranteed to have a valid allocation, and we are taking ownership of it, so it's safe to construct the control block and extent.
let base: NonNull<u8> = unsafe { NonNull::new_unchecked(ptr) };
let ctrlb = ControlBlock::new(base, len, capacity);
Self {
@@ -216,7 +202,6 @@ impl<const ALIGN: usize> From<Owned<ALIGN>> for Frozen<ALIGN> {
ptr: base,
len,
ctrlb,
- _pad: 0,
},
}
}
@@ -226,11 +211,77 @@ impl<const ALIGN: usize> Frozen<ALIGN> {
pub fn as_slice(&self) -> &[u8] {
self.inner.as_slice()
}
+
+ pub fn split_at(self, split_at: usize) -> (Prefix<ALIGN>, Frozen<ALIGN>) {
+ assert!(split_at <= self.inner.len);
+
+ let prefix = Prefix::copy_from_slice(&self.inner.as_slice()[..split_at]);
+
+ let mut tail = self.inner;
+ tail.ptr = unsafe { NonNull::new_unchecked(tail.ptr.as_ptr().add(split_at)) };
+ tail.len -= split_at;
+
+ (prefix, Frozen { inner: tail })
+ }
+
+ pub fn slice(&self, range: impl RangeBounds<usize>) -> Self {
+ use std::ops::Bound;
+
+ let begin = match range.start_bound() {
+ Bound::Included(&n) => n,
+ Bound::Excluded(&n) => n + 1,
+ Bound::Unbounded => 0,
+ };
+
+ let end = match range.end_bound() {
+ Bound::Included(&n) => n.checked_add(1).expect("out of range"),
+ Bound::Excluded(&n) => n,
+ Bound::Unbounded => self.len(),
+ };
+
+ assert!(begin <= self.len());
+ assert!(begin <= end);
+ assert!(end <= self.len());
+
+ let mut sliced = self.clone();
+ let ptr = unsafe { NonNull::new_unchecked(sliced.inner.ptr.as_ptr().add(begin)) };
+ sliced.inner.ptr = ptr;
+ sliced.inner.len = end - begin;
+ sliced
+ }
+
+ pub fn len(&self) -> usize {
+ self.inner.len
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.inner.len == 0
+ }
+}
+
+impl<const ALIGN: usize> std::fmt::Debug for Frozen<ALIGN> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Frozen").field("len", &self.len()).finish()
+ }
+}
+
+impl<const ALIGN: usize> Deref for Frozen<ALIGN> {
+ type Target = [u8];
+
+ fn deref(&self) -> &Self::Target {
+ self.as_slice()
+ }
+}
+
+impl<const ALIGN: usize> IoBuf for Frozen<ALIGN> {
+ fn as_init(&self) -> &[u8] {
+ self.as_slice()
+ }
}
#[repr(C, align(64))]
-struct ControlBlock {
- ref_count: AtomicUsize,
+pub(crate) struct ControlBlock {
+ pub(crate) ref_count: AtomicUsize,
base: NonNull<u8>,
len: usize,
capacity: usize,
@@ -244,41 +295,32 @@ impl ControlBlock {
len,
capacity,
});
- // SAFETY: Box::into_raw returns a valid pointer
unsafe { NonNull::new_unchecked(Box::into_raw(ctrl)) }
}
}
struct Extent<const ALIGN: usize> {
- ptr: NonNull<u8>,
- len: usize,
- ctrlb: NonNull<ControlBlock>,
- // Padded to 32 bytes in order to avoid false sharing when used by `TwoHalves`
- // If `Extent` would be smaller than 32 bytes, two `Extent`s that are adjacent in memory
- // could potentially share the same cache line + some extra
- // that extra could lead to false sharing, in case of invalidation of extra
- _pad: usize,
+ pub(crate) ptr: NonNull<u8>,
+ pub(crate) len: usize,
+ pub(crate) ctrlb: NonNull<ControlBlock>,
}
impl<const ALIGN: usize> Drop for Extent<ALIGN> {
fn drop(&mut self) {
- // SAFETY: `self.ctrlb` points to a live control block while `self` is alive.
unsafe { release_control_block_w_allocation::<ALIGN>(self.ctrlb) }
}
}
impl<const ALIGN: usize> Extent<ALIGN> {
- fn as_slice(&self) -> &[u8] {
- // SAFETY: ptr and len describe a valid allocation
+ pub(crate) fn as_slice(&self) -> &[u8] {
unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) }
}
- unsafe fn as_mut_slice(&mut self) -> &mut [u8] {
- // SAFETY: caller guarantees exclusive access
+ pub(crate) unsafe fn as_mut_slice(&mut self) -> &mut [u8] {
unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) }
}
- fn copy_from_slice(src: &[u8]) -> Self {
+ pub(crate) fn copy_from_slice(src: &[u8]) -> Self {
let mut v: AVec<u8, ConstAlign<ALIGN>> = AVec::new(ALIGN);
v.extend_from_slice(src);
@@ -291,14 +333,12 @@ impl<const ALIGN: usize> Extent<ALIGN> {
ptr: data,
len,
ctrlb,
- _pad: 0,
}
}
}
impl<const ALIGN: usize> Clone for Extent<ALIGN> {
fn clone(&self) -> Self {
- // SAFETY: `self.ctrlb` points to a live control block while `self` is alive.
unsafe {
self.ctrlb
.as_ref()
@@ -309,13 +349,44 @@ impl<const ALIGN: usize> Clone for Extent<ALIGN> {
ptr: self.ptr,
len: self.len,
ctrlb: self.ctrlb,
- _pad: 0,
}
}
}
+pub(crate) fn freeze_extent<const ALIGN: usize>(extent: Extent<ALIGN>) -> Frozen<ALIGN> {
+ Frozen { inner: extent }
+}
+
+pub(crate) fn extent_from_aligned_vec<const ALIGN: usize>(
+ vec: AVec<u8, ConstAlign<ALIGN>>,
+) -> Extent<ALIGN> {
+ let (ptr, _, len, capacity) = vec.into_raw_parts();
+ let data = unsafe { NonNull::new_unchecked(ptr) };
+ let ctrlb = ControlBlock::new(data, len, capacity);
+
+ Extent {
+ ptr: data,
+ len,
+ ctrlb,
+ }
+}
+
+pub(crate) fn extent_offset_from_base<const ALIGN: usize>(extent: &Extent<ALIGN>) -> usize {
+ // SAFETY: the extent points into the control block allocation while it is alive.
+ let offset = unsafe {
+ extent
+ .ptr
+ .as_ptr()
+ .offset_from(extent.ctrlb.as_ref().base.as_ptr())
+ };
+ usize::try_from(offset).expect("extent pointer must not point before control block base")
+}
+
+fn extent_ref_count<const ALIGN: usize>(extent: &Extent<ALIGN>) -> usize {
+ unsafe { extent.ctrlb.as_ref().ref_count.load(Ordering::Acquire) }
+}
+
unsafe fn release_control_block_w_allocation<const ALIGN: usize>(ctrlb: NonNull<ControlBlock>) {
- // SAFETY: ctrlb is valid per function preconditions
let old = unsafe { ctrlb.as_ref() }
.ref_count
.fetch_sub(1, Ordering::Release);
@@ -325,31 +396,10 @@ unsafe fn release_control_block_w_allocation<const ALIGN: usize>(ctrlb: NonNull<
return;
}
- // This fence is needed to prevent reordering of use of the data and
- // deletion of the data. Because it is marked `Release`, the decreasing
- // of the reference count synchronizes with this `Acquire` fence. This
- // means that use of the data happens before decreasing the reference
- // count, which happens before this fence, which happens before the
- // deletion of the data.
- //
- // As explained in the [Boost documentation][1],
- //
- // > It is important to enforce any possible access to the object in one
- // > thread (through an existing reference) to *happen before* deleting
- // > the object in a different thread. This is achieved by a "release"
- // > operation after dropping a reference (any access to the object
- // > through this reference must obviously happened before), and an
- // > "acquire" operation before deleting the object.
- //
- // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
- //
fence(Ordering::Acquire);
- // SAFETY: refcount is zero, we have exclusive ownership
let ctrlb = unsafe { Box::from_raw(ctrlb.as_ptr()) };
- // SAFETY: `ctrlb.base`, `ctrlb.len` and `ctrlb.capacity` were captured from an `AVec`
- // allocation. We reconstruct the AVec and let it deallocate properly.
let _ = unsafe {
AVec::<u8, ConstAlign<ALIGN>>::from_raw_parts(
ctrlb.base.as_ptr(),
@@ -362,243 +412,74 @@ unsafe fn release_control_block_w_allocation<const ALIGN: usize>(ctrlb: NonNull<
unsafe fn reclaim_unique_control_block(ctrlb: NonNull<ControlBlock>) -> ControlBlock {
assert_eq!(
- // SAFETY: caller guarantees `ctrlb` points to a live control block.
unsafe { ctrlb.as_ref() }.ref_count.load(Ordering::Acquire),
1
);
-
- // SAFETY: caller guarantees uniqueness, so ownership of the control block can be reclaimed directly.
unsafe { *Box::from_raw(ctrlb.as_ptr()) }
}
-// TODO: Better tests & miri.
-#[cfg(test)]
-mod tests {
- use super::Owned;
- use aligned_vec::AVec;
- use aligned_vec::ConstAlign;
-
- fn make_owned(data: &[u8]) -> Owned {
- let mut v: AVec<u8, ConstAlign<4096>> = AVec::new(4096);
- v.extend_from_slice(data);
- v.into()
- }
-
- #[test]
- fn split_exposes_head_and_tail() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut buffer = owned.split_at(2);
-
- assert_eq!(buffer.head(), &[1, 2]);
- assert_eq!(buffer.tail(), &[3, 4, 5]);
- assert_eq!(buffer.split_point(), 2);
- assert_eq!(buffer.total_len(), 5);
-
- buffer.head_mut().copy_from_slice(&[9, 8]);
- assert_eq!(buffer.head(), &[9, 8]);
- assert_eq!(buffer.tail(), &[3, 4, 5]);
- }
-
- #[test]
- fn clone_copies_head_and_shares_tail() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut original = owned.split_at(2);
- let mut cloned = original.clone();
-
- original.head_mut().copy_from_slice(&[9, 9]);
- cloned.head_mut().copy_from_slice(&[7, 7]);
-
- assert_eq!(original.head(), &[9, 9]);
- assert_eq!(cloned.head(), &[7, 7]);
- assert_eq!(original.tail(), &[3, 4, 5]);
- assert_eq!(cloned.tail(), &[3, 4, 5]);
- }
-
- #[test]
- fn try_merge_reuses_original_frame_when_unique() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut buffer = owned.split_at(2);
- buffer.head_mut().copy_from_slice(&[8, 9]);
-
- let merged: AVec<u8, ConstAlign<4096>> = buffer.try_merge().unwrap().into();
- assert_eq!(merged.as_slice(), &[8, 9, 3, 4, 5]);
- }
-
- #[test]
- fn try_merge_fails_while_tail_is_shared() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let buffer = owned.split_at(2);
- let clone = buffer.clone();
-
- // Merge fails because tail is shared
- let buffer = buffer.try_merge().unwrap_err();
-
- drop(clone);
-
- // Now merge succeeds
- let merged: AVec<u8, ConstAlign<4096>> = buffer.try_merge().unwrap().into();
- assert_eq!(merged.as_slice(), &[1, 2, 3, 4, 5]);
- }
-
- #[test]
- fn merge_after_cloned_head_mutation_writes_back_to_original_frame() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let buffer = owned.split_at(2);
- let mut clone = buffer.clone();
-
- drop(buffer);
-
- clone.head_mut().copy_from_slice(&[4, 2]);
-
- let merged: AVec<u8, ConstAlign<4096>> = clone.try_merge().unwrap().into();
- assert_eq!(merged.as_slice(), &[4, 2, 3, 4, 5]);
- }
-
- #[test]
- fn zero_length_splits_work() {
- let owned = make_owned(&[1, 2, 3]);
- let left_empty = owned.split_at(0);
- assert_eq!(left_empty.head(), &[]);
- assert_eq!(left_empty.tail(), &[1, 2, 3]);
-
- let owned = make_owned(&[1, 2, 3]);
- let right_empty = owned.split_at(3);
- assert_eq!(right_empty.head(), &[1, 2, 3]);
- assert_eq!(right_empty.tail(), &[]);
- }
-
- #[test]
- fn clone_of_clone_shares_tail() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let original = owned.split_at(2);
- let clone1 = original.clone();
- let _clone2 = clone1.clone();
-
- // All clones share tail, so merge should fail
- assert!(original.try_merge().is_err());
- }
-
- #[test]
- fn owned_as_slice_returns_correct_data() {
- let owned = make_owned(&[10, 20, 30, 40, 50]);
- assert_eq!(owned.as_slice(), &[10, 20, 30, 40, 50]);
- }
-
- #[test]
- fn owned_as_slice_empty_buffer() {
- let owned = make_owned(&[]);
- assert_eq!(owned.as_slice(), &[]);
- }
-
- #[test]
- fn owned_as_mut_slice_allows_modification() {
- let mut owned = make_owned(&[1, 2, 3, 4, 5]);
- let slice = owned.as_mut_slice();
- slice[0] = 100;
- slice[4] = 200;
-
- assert_eq!(owned.as_slice(), &[100, 2, 3, 4, 200]);
- }
-
- #[test]
- fn owned_as_mut_slice_full_overwrite() {
- let mut owned = make_owned(&[1, 2, 3]);
- owned.as_mut_slice().copy_from_slice(&[7, 8, 9]);
-
- assert_eq!(owned.as_slice(), &[7, 8, 9]);
- }
-
- #[test]
- fn owned_modifications_persist_after_split() {
- let mut owned = make_owned(&[1, 2, 3, 4, 5]);
- owned.as_mut_slice()[0] = 99;
- owned.as_mut_slice()[4] = 88;
-
- let buffer = owned.split_at(2);
- assert_eq!(buffer.head(), &[99, 2]);
- assert_eq!(buffer.tail(), &[3, 4, 88]);
- }
-
- #[test]
- fn two_halves_head_returns_correct_slice() {
- let owned = make_owned(&[10, 20, 30, 40, 50]);
- let buffer = owned.split_at(3);
-
- assert_eq!(buffer.head(), &[10, 20, 30]);
- }
-
- #[test]
- fn two_halves_tail_returns_correct_slice() {
- let owned = make_owned(&[10, 20, 30, 40, 50]);
- let buffer = owned.split_at(3);
-
- assert_eq!(buffer.tail(), &[40, 50]);
- }
-
- #[test]
- fn two_halves_head_mut_allows_modification() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut buffer = owned.split_at(3);
-
- buffer.head_mut()[0] = 100;
- buffer.head_mut()[2] = 200;
-
- assert_eq!(buffer.head(), &[100, 2, 200]);
- assert_eq!(buffer.tail(), &[4, 5]);
+unsafe fn try_merge_extents<const ALIGN: usize>(
+ prefix: Extent<ALIGN>,
+ tail: Extent<ALIGN>,
+) -> Result<Owned<ALIGN>, (Extent<ALIGN>, Extent<ALIGN>)> {
+ let split_at = prefix.len;
+ let tail_ctrlb = tail.ctrlb;
+ let prefix_ctrlb = prefix.ctrlb;
+ let ctrlb_eq = std::ptr::addr_eq(prefix_ctrlb.as_ptr(), tail_ctrlb.as_ptr());
+ let ref_count = unsafe { tail_ctrlb.as_ref().ref_count.load(Ordering::Acquire) };
+
+ let is_unique = if ctrlb_eq {
+ ref_count == 2
+ } else {
+ ref_count == 1
+ };
+ if !is_unique {
+ return Err((prefix, tail));
}
- #[test]
- fn two_halves_head_mut_full_overwrite() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut buffer = owned.split_at(3);
+ unsafe {
+ if !ctrlb_eq {
+ let dst_ctrlb = tail_ctrlb.as_ref();
+ let dst = slice::from_raw_parts_mut(dst_ctrlb.base.as_ptr(), split_at);
+ dst.copy_from_slice(prefix.as_slice());
+ }
- buffer.head_mut().copy_from_slice(&[7, 8, 9]);
+ drop(prefix);
+ std::mem::forget(tail);
- assert_eq!(buffer.head(), &[7, 8, 9]);
- assert_eq!(buffer.tail(), &[4, 5]);
+ let ctrlb = reclaim_unique_control_block(tail_ctrlb);
+ let inner = AVec::from_raw_parts(ctrlb.base.as_ptr(), ALIGN, ctrlb.len, ctrlb.capacity);
+ Ok(Owned { inner })
}
+}
- #[test]
- fn two_halves_head_empty_slice() {
- let owned = make_owned(&[1, 2, 3]);
- let buffer = owned.split_at(0);
-
- assert_eq!(buffer.head(), &[]);
- assert_eq!(buffer.tail(), &[1, 2, 3]);
+pub(crate) fn try_coalesce_prefix_with_tail<const ALIGN: usize>(
+ prefix: Extent<ALIGN>,
+ tail: Frozen<ALIGN>,
+) -> Result<Frozen<ALIGN>, (Extent<ALIGN>, Frozen<ALIGN>)> {
+ if !can_coalesce_prefix_with_tail(&prefix, &tail) {
+ return Err((prefix, tail));
}
- #[test]
- fn two_halves_tail_empty_slice() {
- let owned = make_owned(&[1, 2, 3]);
- let buffer = owned.split_at(3);
-
- assert_eq!(buffer.head(), &[1, 2, 3]);
- assert_eq!(buffer.tail(), &[]);
+ match unsafe { try_merge_extents(prefix, tail.inner) } {
+ Ok(owned) => Ok(Frozen::from(owned)),
+ Err((prefix, tail)) => Err((prefix, Frozen { inner: tail })),
}
+}
- #[test]
- fn two_halves_head_mut_does_not_affect_tail() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut buffer = owned.split_at(2);
-
- let original_tail: Vec<u8> = buffer.tail().to_vec();
- buffer.head_mut().copy_from_slice(&[99, 99]);
+fn can_coalesce_prefix_with_tail<const ALIGN: usize>(
+ prefix: &Extent<ALIGN>,
+ tail: &Frozen<ALIGN>,
+) -> bool {
+ let split_at = prefix.len;
+ let tail_ctrlb = tail.inner.ctrlb;
+ let prefix_ctrlb = prefix.ctrlb;
+ let ctrlb_eq = std::ptr::addr_eq(prefix_ctrlb.as_ptr(), tail_ctrlb.as_ptr());
+ let tail_ref_count = extent_ref_count(&tail.inner);
- assert_eq!(buffer.tail(), original_tail.as_slice());
+ if ctrlb_eq {
+ return tail_ref_count == 2;
}
- #[test]
- fn two_halves_cloned_head_mut_independent() {
- let owned = make_owned(&[1, 2, 3, 4, 5]);
- let mut original = owned.split_at(2);
- let mut cloned = original.clone();
-
- original.head_mut().copy_from_slice(&[10, 20]);
- cloned.head_mut().copy_from_slice(&[30, 40]);
-
- assert_eq!(original.head(), &[10, 20]);
- assert_eq!(cloned.head(), &[30, 40]);
- // Tail is shared, both should see the same data
- assert_eq!(original.tail(), cloned.tail());
- }
+ tail_ref_count == 1 && extent_offset_from_base(&tail.inner) == split_at
}
diff --git a/core/journal/Cargo.toml b/core/journal/Cargo.toml
index bd3a0e520..ba80d7a2a 100644
--- a/core/journal/Cargo.toml
+++ b/core/journal/Cargo.toml
@@ -29,9 +29,9 @@ readme = "../../../README.md"
[dependencies]
bytemuck = { workspace = true }
-bytes = { workspace = true }
compio = { workspace = true }
iggy_binary_protocol = { workspace = true }
+iobuf = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
diff --git a/core/journal/src/metadata_journal.rs b/core/journal/src/metadata_journal.rs
index fe45e9f7b..5520e4c8d 100644
--- a/core/journal/src/metadata_journal.rs
+++ b/core/journal/src/metadata_journal.rs
@@ -17,10 +17,10 @@
use crate::file_storage::FileStorage;
use crate::{Journal, JournalHandle};
-use bytes::Bytes;
use compio::io::AsyncWriteAtExt;
use iggy_binary_protocol::consensus::message::Message;
use iggy_binary_protocol::consensus::{Command2, PrepareHeader};
+use iobuf::Owned;
use std::cell::{Cell, Ref, RefCell};
use std::fmt;
use std::io;
@@ -259,8 +259,11 @@ impl MetadataJournal {
};
let buf = vec![0u8; size];
let buf = self.storage.read_at(offset, buf).await?;
- let msg = Message::from_bytes(Bytes::from(buf))
- .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?;
+ let msg = Message::try_from(Owned::<4096>::copy_from_slice(&buf)).map_err(
+ |e: iggy_binary_protocol::consensus::ConsensusError| {
+ io::Error::new(io::ErrorKind::InvalidData, e.to_string())
+ },
+ )?;
Ok(Some(msg))
}
}
@@ -347,8 +350,11 @@ impl Journal<FileStorage> for MetadataJournal {
for (header, offset) in &to_drain {
let buf = vec![0u8; header.size as usize];
let buf = self.storage.read_at(*offset, buf).await?;
- let msg = Message::from_bytes(Bytes::from(buf))
- .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?;
+ let msg = Message::try_from(Owned::<4096>::copy_from_slice(&buf)).map_err(
+ |e: iggy_binary_protocol::consensus::ConsensusError| {
+ io::Error::new(io::ErrorKind::InvalidData, e.to_string())
+ },
+ )?;
drained.push(msg);
}
@@ -406,7 +412,7 @@ impl Journal<FileStorage> for MetadataJournal {
let header = *entry.header();
let offset = self.storage.file_len();
- self.storage.write_append(entry.into_inner()).await?;
+ self.storage.write_append(entry.as_slice().to_vec()).await?;
self.storage.fsync().await?;
let slot = slot_for_op(header.op);
@@ -450,7 +456,7 @@ impl Journal<FileStorage> for MetadataJournal {
let buffer = vec![0u8; size];
let buffer = self.storage.read_at(offset, buffer).await.ok()?;
- Message::from_bytes(Bytes::from(buffer)).ok()
+ Message::try_from(Owned::<4096>::copy_from_slice(&buffer)).ok()
}
}
@@ -467,26 +473,27 @@ impl JournalHandle for MetadataJournal {
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
mod tests {
use super::*;
- use bytes::BytesMut;
use iggy_binary_protocol::consensus::Operation;
use tempfile::tempdir;
fn make_prepare(op: u64, body_size: usize) -> Message<PrepareHeader> {
let total_size = HEADER_SIZE + body_size;
- let mut buffer = BytesMut::zeroed(total_size);
+ let mut buffer = Owned::<4096>::zeroed(total_size);
- let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(&mut buffer[..HEADER_SIZE]);
+ let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(
+ &mut buffer.as_mut_slice()[..HEADER_SIZE],
+ );
header.size = total_size as u32;
header.command = Command2::Prepare;
header.op = op;
header.operation = Operation::CreateStream;
// Fill body with recognizable pattern
- for (i, byte) in buffer[HEADER_SIZE..].iter_mut().enumerate() {
+ for (i, byte) in buffer.as_mut_slice()[HEADER_SIZE..].iter_mut().enumerate() {
*byte = (op as u8).wrapping_add(i as u8);
}
- Message::from_bytes(buffer.freeze()).unwrap()
+ Message::try_from(buffer).unwrap()
}
#[compio::test]
@@ -508,8 +515,8 @@ mod tests {
let msg1 = make_prepare(1, 64);
let msg2 = make_prepare(2, 32);
- journal.append(msg1.clone()).await.unwrap();
- journal.append(msg2.clone()).await.unwrap();
+ journal.append(msg1.deep_copy()).await.unwrap();
+ journal.append(msg2.deep_copy()).await.unwrap();
assert_eq!(journal.last_op(), Some(2));
assert!(journal.header(1).is_some());
@@ -518,11 +525,11 @@ mod tests {
let entry1 = journal.entry(msg1.header()).await.unwrap();
assert_eq!(entry1.header().op, 1);
- assert_eq!(entry1.body().len(), 64);
+ assert_eq!(entry1.as_slice()[HEADER_SIZE..].len(), 64);
let entry2 = journal.entry(msg2.header()).await.unwrap();
assert_eq!(entry2.header().op, 2);
- assert_eq!(entry2.body().len(), 32);
+ assert_eq!(entry2.as_slice()[HEADER_SIZE..].len(), 32);
}
#[compio::test]
@@ -684,7 +691,7 @@ mod tests {
assert_eq!(h.op, op);
let entry = journal.entry_at(&h).await.unwrap().unwrap();
assert_eq!(entry.header().op, op);
- assert_eq!(entry.body().len(), 64);
+ assert_eq!(entry.as_slice()[HEADER_SIZE..].len(), 64);
}
// Reopen and verify the drained WAL is valid
@@ -695,7 +702,7 @@ mod tests {
let h = *journal.header(op as usize).unwrap();
let entry = journal.entry_at(&h).await.unwrap().unwrap();
assert_eq!(entry.header().op, op);
- assert_eq!(entry.body().len(), 64);
+ assert_eq!(entry.as_slice()[HEADER_SIZE..].len(), 64);
}
}
diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs
index efb3a7e6c..5b96b4bcf 100644
--- a/core/message_bus/src/lib.rs
+++ b/core/message_bus/src/lib.rs
@@ -41,12 +41,12 @@ pub trait MessageBus {
&self,
client_id: Self::Client,
data: Self::Data,
- ) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<Self::Data, IggyError>>;
fn send_to_replica(
&self,
replica: Self::Replica,
data: Self::Data,
- ) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<Self::Data, IggyError>>;
}
// TODO: explore generics for Strategy
@@ -113,25 +113,25 @@ impl MessageBus for IggyMessageBus {
async fn send_to_client(
&self,
client_id: Self::Client,
- _message: Self::Data,
- ) -> Result<(), IggyError> {
+ message: Self::Data,
+ ) -> Result<Self::Data, IggyError> {
#[allow(clippy::cast_possible_truncation)] // IggyError::ClientNotFound takes u32
let _sender = self
.clients
.get(&client_id)
.ok_or(IggyError::ClientNotFound(client_id as u32))?;
- Ok(())
+ Ok(message)
}
async fn send_to_replica(
&self,
replica: Self::Replica,
- _message: Self::Data,
- ) -> Result<(), IggyError> {
+ message: Self::Data,
+ ) -> Result<Self::Data, IggyError> {
// TODO: Handle lazily creating the connection.
let _connection = self
.get_replica_connection(replica)
.ok_or(IggyError::ResourceNotFound(format!("Replica {replica}")))?;
- Ok(())
+ Ok(message)
}
}
diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml
index 81f88e7f2..97f7ae997 100644
--- a/core/metadata/Cargo.toml
+++ b/core/metadata/Cargo.toml
@@ -47,6 +47,7 @@ tracing = { workspace = true }
bytemuck = { workspace = true }
bytes = { workspace = true }
compio = { workspace = true }
+iobuf = { workspace = true }
tempfile = { workspace = true }
[lints.clippy]
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 820ae6649..484135f19 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -17,8 +17,9 @@
use crate::stm::StateMachine;
use crate::stm::snapshot::{FillSnapshot, MetadataSnapshot, Snapshot, SnapshotError};
use consensus::{
- Consensus, Pipeline, PipelineEntry, Plane, PlaneIdentity, Project, Sequencer, VsrConsensus,
- ack_preflight, ack_quorum_reached, build_reply_message, drain_committable_prefix,
+ CommitLogEvent, Consensus, Pipeline, PipelineEntry, Plane, PlaneIdentity, PlaneKind, Project,
+ ReplicaLogContext, RequestLogEvent, Sequencer, SimEventKind, VsrConsensus, ack_preflight,
+ ack_quorum_reached, build_reply_message, drain_committable_prefix, emit_sim_event,
fence_old_prepare_by_commit, panic_if_hash_chain_would_break_in_same_view,
pipeline_prepare_common, replicate_preflight, replicate_to_next_in_chain,
send_prepare_ok as send_prepare_ok_common,
@@ -32,6 +33,10 @@ use message_bus::MessageBus;
use std::path::Path;
use tracing::{debug, error, warn};
+const fn freeze_client_reply(message: Message<GenericHeader>) -> Message<GenericHeader> {
+ message
+}
+
#[derive(Debug, Clone)]
#[allow(unused)]
pub struct IggySnapshot {
@@ -287,24 +292,40 @@ where
async fn on_request(&self, message: <VsrConsensus<B> as Consensus>::Message<RequestHeader>) {
let consensus = self.consensus.as_ref().unwrap();
- // TODO: Bunch of asserts.
- debug!("handling metadata request");
+ emit_sim_event(
+ SimEventKind::ClientRequestReceived,
+ &RequestLogEvent {
+ replica: ReplicaLogContext::from_consensus(consensus, PlaneKind::Metadata),
+ client_id: message.header().client,
+ request_id: message.header().request,
+ operation: message.header().operation,
+ },
+ );
let prepare = message.project(consensus);
- pipeline_prepare_common(consensus, prepare, |prepare| self.on_replicate(prepare)).await;
+ pipeline_prepare_common(consensus, PlaneKind::Metadata, prepare, |prepare| {
+ self.on_replicate(prepare)
+ })
+ .await;
}
async fn on_replicate(&self, message: <VsrConsensus<B> as Consensus>::Message<PrepareHeader>) {
let consensus = self.consensus.as_ref().unwrap();
let journal = self.journal.as_ref().unwrap();
- let header = message.header();
+ let header = *message.header();
- let current_op = match replicate_preflight(consensus, header) {
+ let current_op = match replicate_preflight(consensus, &header) {
Ok(current_op) => current_op,
Err(reason) => {
warn!(
- replica = consensus.replica(),
- "on_replicate: ignoring ({reason})"
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ operation = ?header.operation,
+ reason = reason.as_str(),
+ "ignoring prepare during replicate preflight"
);
return;
}
@@ -312,13 +333,23 @@ where
// TODO: Handle idx calculation, for now using header.op, but since the journal may get compacted, this may not be correct.
#[allow(clippy::cast_possible_truncation)]
- let is_old_prepare = fence_old_prepare_by_commit(consensus, header)
+ let is_old_prepare = fence_old_prepare_by_commit(consensus, &header)
|| journal.handle().header(header.op as usize).is_some();
- if is_old_prepare {
- warn!("received old prepare, not replicating");
+ let message = if is_old_prepare {
+ warn!(
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ commit = consensus.commit(),
+ operation = ?header.operation,
+ "received old prepare, skipping replication"
+ );
+ message
} else {
- self.replicate(message.clone()).await;
- }
+ self.replicate(message).await
+ };
// TODO add assertions for valid state here.
@@ -333,15 +364,22 @@ where
{
Ok(true) => {
debug!(
- replica = consensus.replica(),
- "on_replicate: forced checkpoint at op={snap_op}"
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ checkpoint_op = snap_op,
+ "forced checkpoint completed"
);
}
Ok(false) => {}
Err(e) => {
error!(
- replica = consensus.replica(),
- "on_replicate: forced checkpoint failed: {e}"
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ checkpoint_op = snap_op,
+ error = %e,
+ "forced checkpoint failed"
);
return;
}
@@ -349,8 +387,8 @@ where
}
// Verify hash chain integrity.
- if let Some(previous) = journal.handle().previous_header(header) {
- panic_if_hash_chain_would_break_in_same_view(&previous, header);
+ if let Some(previous) = journal.handle().previous_header(&header) {
+ panic_if_hash_chain_would_break_in_same_view(&previous, &header);
}
assert_eq!(header.op, current_op + 1);
@@ -359,16 +397,21 @@ where
consensus.set_last_prepare_checksum(header.checksum);
// Append to journal.
- if let Err(e) = journal.handle().append(message.clone()).await {
+ if let Err(e) = journal.handle().append(message).await {
error!(
- replica = consensus.replica(),
- "on_replicate: journal append failed: {e}"
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ op = header.op,
+ operation = ?header.operation,
+ error = %e,
+ "journal append failed"
);
return;
}
// After successful journal write, send prepare_ok to primary.
- self.send_prepare_ok(header).await;
+ self.send_prepare_ok(&header).await;
// If follower, commit any newly committable entries.
if consensus.is_follower() {
@@ -381,33 +424,57 @@ where
let header = message.header();
if let Err(reason) = ack_preflight(consensus) {
- warn!("on_ack: ignoring ({reason})");
+ warn!(
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ reason = reason.as_str(),
+ "ignoring ack during preflight"
+ );
return;
}
{
let pipeline = consensus.pipeline().borrow();
if pipeline
- .message_by_op_and_checksum(header.op, header.prepare_checksum)
+ .entry_by_op_and_checksum(header.op, header.prepare_checksum)
.is_none()
{
- debug!("on_ack: prepare not in pipeline op={}", header.op);
+ debug!(
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ op = header.op,
+ prepare_checksum = header.prepare_checksum,
+ "ack target prepare not in pipeline"
+ );
return;
}
}
- if ack_quorum_reached(consensus, header) {
+ if ack_quorum_reached(consensus, PlaneKind::Metadata, header) {
let journal = self.journal.as_ref().unwrap();
- debug!("on_ack: quorum received for op={}", header.op);
+ debug!(
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ op = header.op,
+ "ack quorum received"
+ );
let drained = drain_committable_prefix(consensus);
if let (Some(first), Some(last)) = (drained.first(), drained.last()) {
debug!(
- "on_ack: draining committed prefix ops=[{}..={}] count={}",
- first.header.op,
- last.header.op,
- drained.len()
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ first_op = first.header.op,
+ last_op = last.header.op,
+ drained_count = drained.len(),
+ "draining committed metadata prefix"
);
}
@@ -429,24 +496,36 @@ where
let response = self.mux_stm.update(prepare).unwrap_or_else(|err| {
warn!(
- "on_ack: state machine error for op={}: {err}",
- prepare_header.op
+ target: "iggy.metadata.diag",
+ plane = "metadata",
+ replica_id = consensus.replica(),
+ op = prepare_header.op,
+ operation = ?prepare_header.operation,
+ error = %err,
+ "state machine update failed for committed metadata entry"
);
bytes::Bytes::new()
});
- debug!("on_ack: state applied for op={}", prepare_header.op);
+ let pipeline_depth = consensus.pipeline().borrow().len();
+ let event = CommitLogEvent {
+ replica: ReplicaLogContext::from_consensus(consensus, PlaneKind::Metadata),
+ op: prepare_header.op,
+ client_id: prepare_header.client,
+ request_id: prepare_header.request,
+ operation: prepare_header.operation,
+ pipeline_depth,
+ };
+ emit_sim_event(SimEventKind::OperationCommitted, &event);
let generic_reply =
build_reply_message(consensus, &prepare_header, response).into_generic();
- debug!(
- "on_ack: sending reply to client={} for op={}",
- prepare_header.client, prepare_header.op
- );
+ let reply_buffers = freeze_client_reply(generic_reply);
+ emit_sim_event(SimEventKind::ClientReplyEmitted, &event);
// TODO: Propagate send error instead of panicking; requires bus error design.
consensus
.message_bus()
- .send_to_client(prepare_header.client, generic_reply)
+ .send_to_client(prepare_header.client, reply_buffers)
.await
.unwrap();
}
@@ -457,7 +536,7 @@ where
impl<B, P, J, S, M> PlaneIdentity<VsrConsensus<B, P>> for IggyMetadata<VsrConsensus<B, P>, J, S, M>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
J: JournalHandle,
J::Target: Journal<J::Storage, Entry = Message<PrepareHeader>, Header = PrepareHeader>,
M: StateMachine<Input = Message<PrepareHeader>>,
@@ -477,7 +556,7 @@ where
impl<B, P, J, S, M> IggyMetadata<VsrConsensus<B, P>, J, S, M>
where
B: MessageBus<Replica = u8, Data = Message<GenericHeader>, Client = u128>,
- P: Pipeline<Message = Message<PrepareHeader>, Entry = PipelineEntry>,
+ P: Pipeline<Entry = PipelineEntry>,
J: JournalHandle,
J::Target: Journal<J::Storage, Entry = Message<PrepareHeader>, Header = PrepareHeader>,
M: StateMachine<Input = Message<PrepareHeader>>,
@@ -489,11 +568,11 @@ where
/// - Each backup forwards to the next
/// - Stops when we would forward back to primary
#[allow(clippy::future_not_send)]
- async fn replicate(&self, message: Message<PrepareHeader>) {
+ async fn replicate(&self, message: Message<PrepareHeader>) -> Message<PrepareHeader> {
let consensus = self.consensus.as_ref().unwrap();
let journal = self.journal.as_ref().unwrap();
- let header = message.header();
+ let header = *message.header();
// TODO: calculate the index;
#[allow(clippy::cast_possible_truncation)]
@@ -503,7 +582,7 @@ where
journal.handle().header(idx).is_none(),
"replicate: must not already have prepare"
);
- replicate_to_next_in_chain(consensus, message).await;
+ replicate_to_next_in_chain(consensus, message).await
}
// TODO: Implement jump_to_newer_op
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs
index ef461f18b..d4f01dc6a 100644
--- a/core/metadata/src/impls/recovery.rs
+++ b/core/metadata/src/impls/recovery.rs
@@ -158,8 +158,8 @@ where
#[allow(clippy::cast_possible_truncation)]
mod tests {
use super::*;
- use bytes::BytesMut;
use iggy_binary_protocol::consensus::{Command2, Operation};
+ use iobuf::Owned;
use journal::Journal;
use tempfile::tempdir;
@@ -171,13 +171,15 @@ mod tests {
fn make_prepare(op: u64, body_size: usize) -> Message<PrepareHeader> {
let total_size = HEADER_SIZE + body_size;
- let mut buffer = BytesMut::zeroed(total_size);
- let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(&mut buffer[..HEADER_SIZE]);
+ let mut buffer = Owned::<4096>::zeroed(total_size);
+ let header = bytemuck::checked::from_bytes_mut::<PrepareHeader>(
+ &mut buffer.as_mut_slice()[..HEADER_SIZE],
+ );
header.size = total_size as u32;
header.command = Command2::Prepare;
header.op = op;
header.operation = Operation::CreateStream;
- Message::from_bytes(buffer.freeze()).unwrap()
+ Message::try_from(buffer).unwrap()
}
#[compio::test]
diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs
index 68f36f3a0..0e2bb9b4b 100644
--- a/core/metadata/src/stm/mod.rs
+++ b/core/metadata/src/stm/mod.rs
@@ -247,11 +247,15 @@ macro_rules! collect_handlers {
fn parse(input: Self::Input) -> Result<::iggy_common::Either<Self::Cmd, Self::Input>, Self::Error> {
use ::iggy_common::BytesSerializable;
use ::iggy_common::Either;
- use ::iggy_binary_protocol::Operation;
+ use ::iggy_binary_protocol::{Operation, PrepareHeader};
match input.header().operation {
$(
Operation::$operation => {
- let body = input.body_bytes();
+ // TODO: FIXME, zero allocation operation construction.
+ let header = *input.header();
+ let body = ::bytes::Bytes::copy_from_slice(
+ &input.as_slice()[core::mem::size_of::<PrepareHeader>()..header.size as usize]
+ );
let cmd = $operation::from_bytes(body)?;
Ok(Either::Left([<$state Command>]::$operation(cmd)))
},
diff --git a/core/partitions/Cargo.toml b/core/partitions/Cargo.toml
index 2362abb73..8969e07c0 100644
--- a/core/partitions/Cargo.toml
+++ b/core/partitions/Cargo.toml
@@ -28,13 +28,17 @@ repository = "https://github.com/apache/iggy"
readme = "../../README.md"
[dependencies]
+bytemuck = { workspace = true }
bytes = { workspace = true }
+compio = { workspace = true }
consensus = { workspace = true }
iggy_binary_protocol = { workspace = true }
iggy_common = { workspace = true }
+iobuf = { workspace = true }
journal = { workspace = true }
message_bus = { workspace = true }
ringbuffer = { workspace = true }
+smallvec = "1.13"
tokio = { workspace = true }
tracing = { workspace = true }
diff --git a/core/partitions/src/iggy_index.rs b/core/partitions/src/iggy_index.rs
new file mode 100644
index 000000000..d3cd90aa2
--- /dev/null
+++ b/core/partitions/src/iggy_index.rs
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub const IGGY_INDEX_SIZE: usize = std::mem::size_of::<u64>() * 3;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct IggyIndex {
+ pub offset: u64,
+ pub timestamp: u64,
+ pub position: u64,
+}
+
+impl IggyIndex {
+ #[must_use]
+ pub const fn new(offset: u64, timestamp: u64, position: u64) -> Self {
+ Self {
+ offset,
+ timestamp,
+ position,
+ }
+ }
+
+ fn write_to(&self, buffer: &mut Vec<u8>) {
+ buffer.extend_from_slice(&self.offset.to_le_bytes());
+ buffer.extend_from_slice(&self.timestamp.to_le_bytes());
+ buffer.extend_from_slice(&self.position.to_le_bytes());
+ }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct IggyIndexCache {
+ entries: Vec<IggyIndex>,
+}
+
+impl IggyIndexCache {
+ #[must_use]
+ pub fn empty() -> Self {
+ Self::default()
+ }
+
+ #[must_use]
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ entries: Vec::with_capacity(capacity),
+ }
+ }
+
+ #[must_use]
+ pub const fn len(&self) -> usize {
+ self.entries.len()
+ }
+
+ #[must_use]
+ pub const fn is_empty(&self) -> bool {
+ self.entries.is_empty()
+ }
+
+ pub fn insert(&mut self, offset: u64, timestamp: u64, position: u64) {
+ self.entries
+ .push(IggyIndex::new(offset, timestamp, position));
+ }
+
+ #[must_use]
+ pub fn serialize(indexes: &IggyIndex) -> Vec<u8> {
+ let mut buffer = Vec::with_capacity(IGGY_INDEX_SIZE);
+ indexes.write_to(&mut buffer);
+ buffer
+ }
+
+ #[must_use]
+ pub fn offset_lower_bound(&self, offset: u64) -> Option<&IggyIndex> {
+ match self
+ .entries
+ .binary_search_by_key(&offset, |entry| entry.offset)
+ {
+ Ok(idx) => self.entries.get(idx),
+ Err(0) => self.entries.first(),
+ Err(idx) => self.entries.get(idx - 1),
+ }
+ }
+
+ #[must_use]
+ pub fn timestamp_lower_bound(&self, timestamp: u64) -> Option<&IggyIndex> {
+ match self
+ .entries
+ .binary_search_by_key(×tamp, |entry| entry.timestamp)
+ {
+ Ok(idx) => self.entries.get(idx),
+ Err(0) => self.entries.first(),
+ Err(idx) => self.entries.get(idx - 1),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{IggyIndex, IggyIndexCache};
+
+ fn cache() -> IggyIndexCache {
+ let mut cache = IggyIndexCache::empty();
+ cache.insert(10, 100, 1);
+ cache.insert(20, 200, 2);
+ cache.insert(30, 300, 3);
+ cache
+ }
+
+ #[test]
+ fn offset_lookup_returns_predecessor() {
+ let cache = cache();
+
+ assert_eq!(
+ cache.offset_lower_bound(25),
+ Some(&IggyIndex::new(20, 200, 2))
+ );
+ assert_eq!(
+ cache.offset_lower_bound(20),
+ Some(&IggyIndex::new(20, 200, 2))
+ );
+ assert_eq!(
+ cache.offset_lower_bound(5),
+ Some(&IggyIndex::new(10, 100, 1))
+ );
+ assert_eq!(
+ cache.offset_lower_bound(35),
+ Some(&IggyIndex::new(30, 300, 3))
+ );
+ }
+
+ #[test]
+ fn timestamp_lookup_returns_predecessor() {
+ let cache = cache();
+
+ assert_eq!(
+ cache.timestamp_lower_bound(250),
+ Some(&IggyIndex::new(20, 200, 2))
+ );
+ assert_eq!(
+ cache.timestamp_lower_bound(200),
+ Some(&IggyIndex::new(20, 200, 2))
+ );
+ assert_eq!(
+ cache.timestamp_lower_bound(50),
+ Some(&IggyIndex::new(10, 100, 1))
+ );
+ assert_eq!(
+ cache.timestamp_lower_bound(350),
+ Some(&IggyIndex::new(30, 300, 3))
+ );
+ }
+}
diff --git a/core/partitions/src/iggy_index_writer.rs b/core/partitions/src/iggy_index_writer.rs
new file mode 100644
index 000000000..3aa05ac5b
--- /dev/null
+++ b/core/partitions/src/iggy_index_writer.rs
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use compio::fs::{File, OpenOptions};
+use compio::io::AsyncWriteAtExt;
+use iggy_common::IggyError;
+use std::rc::Rc;
+use std::sync::atomic::{AtomicU64, Ordering};
+use tracing::trace;
+
+#[derive(Debug)]
+pub struct IggyIndexWriter {
+ file_path: String,
+ file: File,
+ index_size_bytes: Rc<AtomicU64>,
+ fsync: bool,
+}
+
+impl IggyIndexWriter {
+ pub async fn new(
+ file_path: &str,
+ index_size_bytes: Rc<AtomicU64>,
+ fsync: bool,
+ file_exists: bool,
+ ) -> Result<Self, IggyError> {
+ let mut opts = OpenOptions::new();
+ opts.create(true).write(true);
+ let file = opts
+ .open(file_path)
+ .await
+ .map_err(|_| IggyError::CannotReadFile)?;
+
+ if file_exists {
+ let _ = file.sync_all().await;
+
+ let actual_index_size = file
+ .metadata()
+ .await
+ .map_err(|_| IggyError::CannotReadFileMetadata)?
+ .len();
+
+ index_size_bytes.store(actual_index_size, Ordering::Relaxed);
+ }
+
+ let size = index_size_bytes.load(Ordering::Relaxed);
+ trace!(
+ target: "iggy.partitions.storage",
+ file = file_path,
+ size,
+ "opened sparse index file for writing"
+ );
+
+ Ok(Self {
+ file_path: file_path.to_owned(),
+ file,
+ index_size_bytes,
+ fsync,
+ })
+ }
+
+ pub async fn save_indexes(&self, indexes: Vec<u8>) -> Result<(), IggyError> {
+ if indexes.is_empty() {
+ return Ok(());
+ }
+
+ let len = indexes.len();
+ let position = self.index_size_bytes.load(Ordering::Relaxed);
+ let file = &self.file;
+ (&*file)
+ .write_all_at(indexes, position)
+ .await
+ .0
+ .map_err(|_| IggyError::CannotSaveIndexToSegment)?;
+
+ self.index_size_bytes
+ .fetch_add(len as u64, Ordering::Release);
+
+ if self.fsync {
+ self.fsync().await?;
+ }
+
+ trace!(
+ target: "iggy.partitions.storage",
+ file = self.file_path.as_str(),
+ bytes = len,
+ position,
+ "saved sparse index bytes to file"
+ );
+ Ok(())
+ }
+
+ pub async fn fsync(&self) -> Result<(), IggyError> {
+ self.file
+ .sync_all()
+ .await
+ .map_err(|_| IggyError::CannotWriteToFile)?;
+ Ok(())
+ }
+}
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 31bc3ba53..c49911b3d 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -20,14 +20,14 @@ use crate::journal::{
};
use crate::log::SegmentedLog;
use crate::{
- AppendResult, Partition, PartitionOffsets, PollingArgs, PollingConsumer,
- decode_send_messages_batch,
+ AppendResult, Partition, PartitionOffsets, PollFragments, PollQueryResult, PollingArgs,
+ PollingConsumer,
};
use iggy_binary_protocol::{Message, Operation, PrepareHeader};
use iggy_common::{
ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets,
- IggyByteSize, IggyError, IggyMessagesBatchMut, IggyMessagesBatchSet, IggyTimestamp,
- PartitionStats, PollingKind,
+ IggyByteSize, IggyError, IggyTimestamp, PartitionStats, PollingKind,
+ send_messages2::stamp_prepare_for_persistence,
};
use journal::Journal as _;
use std::sync::Arc;
@@ -36,13 +36,14 @@ use tokio::sync::Mutex as TokioMutex;
use tracing::warn;
// This struct aliases in terms of the code contained the `LocalPartition from `core/server/src/streaming/partitions/local_partition.rs`.
+//
+// TODO: Fix op deduplication once we move to a consensus-per-partition design.
#[derive(Debug)]
pub struct IggyPartition {
pub log: SegmentedLog<PartitionJournal<PartitionJournalMemStorage>, PartitionJournalMemStorage>,
- /// Committed offset — advanced only after quorum ack.
+ /// Highest durably persisted offset.
pub offset: Arc<AtomicU64>,
- /// Dirty offset — advanced on every prepare (before commit).
- /// Used to assign offsets during `prepare_for_persistence`.
+ /// Highest offset assigned to prepares that may still only live in the in-memory journal.
pub dirty_offset: AtomicU64,
pub consumer_offsets: Arc<ConsumerOffsets>,
pub consumer_group_offsets: Arc<ConsumerGroupOffsets>,
@@ -54,36 +55,6 @@ pub struct IggyPartition {
}
impl IggyPartition {
- fn prepare_message_from_batch(
- mut header: PrepareHeader,
- batch: &IggyMessagesBatchMut,
- ) -> Message<PrepareHeader> {
- let indexes = batch.indexes();
- let count = batch.count();
- let body_len = 4 + indexes.len() + batch.len();
- let total_size = std::mem::size_of::<PrepareHeader>() + body_len;
- header.size = u32::try_from(total_size)
- .expect("prepare_message_from_batch: batch size exceeds u32::MAX");
-
- let message = Message::<PrepareHeader>::new(total_size).transmute_header(|_old, new| {
- *new = header;
- });
-
- let mut bytes = message
- .into_inner()
- .try_into_mut()
- .expect("prepare_message_from_batch: expected unique bytes buffer");
- let header_size = std::mem::size_of::<PrepareHeader>();
- bytes[header_size..header_size + 4].copy_from_slice(&count.to_le_bytes());
- let mut position = header_size + 4;
- bytes[position..position + indexes.len()].copy_from_slice(indexes);
- position += indexes.len();
- bytes[position..position + batch.len()].copy_from_slice(batch.as_bytes());
-
- Message::<PrepareHeader>::from_bytes(bytes.freeze())
- .expect("prepare_message_from_batch: invalid prepared message bytes")
- }
-
pub fn new(stats: Arc<PartitionStats>) -> Self {
Self {
log: SegmentedLog::default(),
@@ -110,35 +81,27 @@ impl Partition for IggyPartition {
return Err(IggyError::CannotAppendMessage);
}
- let mut batch = decode_send_messages_batch(message.body_bytes())
- .ok_or(IggyError::CannotAppendMessage)?;
-
- if batch.count() == 0 {
- return Ok(AppendResult::new(0, 0, 0));
- }
-
let dirty_offset = if self.should_increment_offset {
self.dirty_offset.load(Ordering::Relaxed) + 1
} else {
0
};
- let segment = self.log.active_segment();
- let segment_start_offset = segment.start_offset;
- let current_position = segment.current_position;
+ // TODO: Replace this with monotonic broker timestamp assignment. If wall clock
+ // time goes backwards, clamp to the partition/log max timestamp instead.
+ let batch_timestamp = IggyTimestamp::now().as_micros();
+ let (message, batch, batch_messages_count) =
+ stamp_prepare_for_persistence(message, dirty_offset, batch_timestamp)
+ .map_err(|_| IggyError::CannotAppendMessage)?;
- batch
- .prepare_for_persistence(segment_start_offset, dirty_offset, current_position, None)
- .await;
+ if batch_messages_count == 0 {
+ return Ok(AppendResult::new(0, 0, 0));
+ }
- let batch_messages_count = batch.count();
- let batch_messages_size = batch.size();
+ let batch_messages_size =
+ u64::try_from(batch.total_size()).map_err(|_| IggyError::CannotAppendMessage)?;
- let last_dirty_offset = if batch_messages_count == 0 {
- dirty_offset
- } else {
- dirty_offset + u64::from(batch_messages_count) - 1
- };
+ let last_dirty_offset = dirty_offset + u64::from(batch_messages_count) - 1;
if !self.should_increment_offset {
self.should_increment_offset = true;
@@ -147,27 +110,25 @@ impl Partition for IggyPartition {
.store(last_dirty_offset, Ordering::Relaxed);
let segment_index = self.log.segments().len() - 1;
- self.log.segments_mut()[segment_index].current_position += batch_messages_size;
+ let current_position = self.log.segments()[segment_index].current_position;
+ self.log.segments_mut()[segment_index].current_position = current_position
+ .checked_add(batch_messages_size)
+ .ok_or(IggyError::CannotAppendMessage)?;
let journal = self.log.journal_mut();
journal.info.messages_count += batch_messages_count;
- journal.info.size += IggyByteSize::from(u64::from(batch_messages_size));
+ journal.info.size += IggyByteSize::from(batch_messages_size);
journal.info.current_offset = last_dirty_offset;
- if let Some(ts) = batch.first_timestamp()
- && journal.info.first_timestamp == 0
- {
- journal.info.first_timestamp = ts;
- }
- if let Some(ts) = batch.last_timestamp() {
- journal.info.end_timestamp = ts;
+ if journal.info.first_timestamp == 0 {
+ journal.info.first_timestamp = batch.base_timestamp;
}
-
- let message = Self::prepare_message_from_batch(header, &batch);
+ journal.info.end_timestamp = batch.base_timestamp;
+ journal.info.max_timestamp = journal.info.max_timestamp.max(batch.base_timestamp);
journal
.inner
- .append(message)
+ .append(message.into_frozen())
.await
- .map_err(|e| IggyError::IoError(e.to_string()))?;
+ .map_err(|_| IggyError::CannotAppendMessage)?;
Ok(AppendResult::new(
dirty_offset,
@@ -180,66 +141,60 @@ impl Partition for IggyPartition {
&self,
consumer: PollingConsumer,
args: PollingArgs,
- ) -> Result<IggyMessagesBatchSet, IggyError> {
+ ) -> Result<PollQueryResult<4096>, IggyError> {
if !self.should_increment_offset || args.count == 0 {
- return Ok(IggyMessagesBatchSet::empty());
+ return Ok((PollFragments::new(), None));
}
- let committed_offset = self.offset.load(Ordering::Relaxed);
+ let write_offset = self.offset.load(Ordering::Acquire);
- let start_offset = match args.strategy.kind {
- PollingKind::Offset => args.strategy.value,
- PollingKind::First => 0,
- PollingKind::Last => committed_offset.saturating_sub(u64::from(args.count) - 1),
+ let result = match args.strategy.kind {
PollingKind::Timestamp => {
- let result = self
- .log
+ self.log
.journal()
.inner
.get(&MessageLookup::Timestamp {
timestamp: args.strategy.value,
count: args.count,
})
- .await;
- let batch_set = result.unwrap_or_else(IggyMessagesBatchSet::empty);
- if let Some(first) = batch_set.first_offset() {
- if first > committed_offset {
- return Ok(IggyMessagesBatchSet::empty());
- }
- let max_count = u32::try_from(committed_offset - first + 1).unwrap_or(u32::MAX);
- return Ok(batch_set.get_by_offset(first, batch_set.count().min(max_count)));
+ .await
+ }
+ kind => {
+ let start_offset = match kind {
+ PollingKind::Offset => args.strategy.value,
+ PollingKind::First => 0,
+ PollingKind::Last => write_offset.saturating_sub(u64::from(args.count) - 1),
+ PollingKind::Next => self
+ .get_consumer_offset(consumer)
+ .map_or(0, |offset| offset + 1),
+ PollingKind::Timestamp => unreachable!(),
+ };
+
+ if start_offset > write_offset {
+ return Ok((PollFragments::new(), None));
}
- return Ok(batch_set);
+
+ self.log
+ .journal()
+ .inner
+ .get(&MessageLookup::Offset {
+ offset: start_offset,
+ count: args.count,
+ })
+ .await
}
- PollingKind::Next => self
- .get_consumer_offset(consumer)
- .map_or(0, |offset| offset + 1),
};
- if start_offset > committed_offset {
- return Ok(IggyMessagesBatchSet::empty());
- }
-
- let max_count = u32::try_from(committed_offset - start_offset + 1).unwrap_or(u32::MAX);
- let count = args.count.min(max_count);
-
- let result = self
- .log
- .journal()
- .inner
- .get(&MessageLookup::Offset {
- offset: start_offset,
- count,
- })
- .await;
-
- let batch_set = result.unwrap_or_else(IggyMessagesBatchSet::empty);
+ let (fragments, last_matching_offset) =
+ result.unwrap_or_else(|| (PollFragments::new(), None));
- if args.auto_commit && !batch_set.is_empty() {
- let last_offset = start_offset + u64::from(batch_set.count()) - 1;
+ if args.auto_commit && !fragments.is_empty() {
+ let last_offset =
+ last_matching_offset.expect("non-empty poll result must have a last offset");
if let Err(err) = self.store_consumer_offset(consumer, last_offset) {
// warning for now.
warn!(
+ target: "iggy.partitions.diag",
consumer = ?consumer,
last_offset,
%err,
@@ -248,7 +203,7 @@ impl Partition for IggyPartition {
}
}
- Ok(batch_set)
+ Ok((fragments, last_matching_offset))
}
#[allow(clippy::cast_possible_truncation)]
@@ -312,7 +267,7 @@ impl Partition for IggyPartition {
fn offsets(&self) -> PartitionOffsets {
PartitionOffsets::new(
- self.offset.load(Ordering::Relaxed),
+ self.offset.load(Ordering::Acquire),
self.dirty_offset.load(Ordering::Relaxed),
)
}
diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs
index 4fb9fc34f..5d9d3d601 100644
--- a/core/partitions/src/iggy_partitions.rs
+++ b/core/partitions/src/iggy_partitions.rs
@@ -20,26 +20,34 @@
use crate::IggyPartition;
use crate::Partition;
use crate::PollingConsumer;
+use crate::iggy_index_writer::IggyIndexWriter;
use crate::log::JournalInfo;
+use crate::messages_writer::MessagesWriter;
+use crate::segment::Segment;
use crate::types::PartitionsConfig;
use consensus::PlaneIdentity;
use consensus::{
- Consensus, NamespacedPipeline, Pipeline, PipelineEntry, Plane, Project, Sequencer,
- VsrConsensus, ack_preflight, build_reply_message, fence_old_prepare_by_commit,
- pipeline_prepare_common, replicate_preflight, replicate_to_next_in_chain,
- send_prepare_ok as send_prepare_ok_common,
+ CommitLogEvent, Consensus, NamespacedPipeline, Pipeline, PipelineEntry, Plane, PlaneKind,
+ Project, ReplicaLogContext, RequestLogEvent, Sequencer, SimEventKind, VsrConsensus,
+ ack_preflight, build_reply_message, emit_namespace_progress_event, emit_sim_event,
+ fence_old_prepare_by_commit, pipeline_prepare_common, replicate_preflight,
+ replicate_to_next_in_chain, send_prepare_ok as send_prepare_ok_common,
};
use iggy_binary_protocol::{
Command2, ConsensusHeader, GenericHeader, Message, Operation, PrepareHeader, PrepareOkHeader,
RequestHeader,
};
use iggy_common::{
- IggyByteSize, PartitionStats, Segment, SegmentStorage,
+ IggyByteSize, IggyError, PartitionStats, SegmentStorage,
+ send_messages2::{convert_request_message, decode_prepare_slice},
sharding::{IggyNamespace, LocalIdx, ShardId},
};
+use iobuf::Frozen;
+use journal::Journal as _;
use message_bus::MessageBus;
use std::cell::UnsafeCell;
use std::collections::{HashMap, HashSet};
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use tracing::{debug, warn};
@@ -61,11 +69,20 @@ pub struct IggyPartitions<C> {
/// Wrapped in `UnsafeCell` for interior mutability — matches the single-threaded
/// per-shard execution model. Consensus trait methods take `&self` but need to
/// mutate partition state (segments, offsets, journal).
+ ///
+ /// TODO: Move to more granular synchronization around partition substate and
+ /// stop exposing `&mut IggyPartition` from `&self`. The long-term shape here
+ /// should let append/commit paths coordinate on the exact mutable state they
+ /// need instead of relying on `UnsafeCell` aliasing for the whole partition.
partitions: UnsafeCell<Vec<IggyPartition>>,
namespace_to_local: HashMap<IggyNamespace, LocalIdx>,
consensus: Option<C>,
}
+const fn freeze_client_reply(message: Message<GenericHeader>) -> Message<GenericHeader> {
+ message
+}
+
impl<C> IggyPartitions<C> {
#[must_use]
pub fn new(shard_id: ShardId, config: PartitionsConfig) -> Self {
@@ -98,13 +115,6 @@ impl<C> IggyPartitions<C> {
unsafe { &*self.partitions.get() }
}
- // TODO: Figure out better way to do this, maybe inline the mutable access to ignore the lint.
- #[allow(clippy::mut_from_ref)]
- fn partitions_mut(&self) -> &mut Vec<IggyPartition> {
- // Safety: single-threaded per-shard model, no concurrent access.
- unsafe { &mut *self.partitions.get() }
- }
-
pub const fn shard_id(&self) -> ShardId {
self.shard_id
}
@@ -123,8 +133,10 @@ impl<C> IggyPartitions<C> {
}
/// Get mutable partition by local index.
- pub fn get_mut(&self, local_idx: LocalIdx) -> Option<&mut IggyPartition> {
- self.partitions_mut().get_mut(*local_idx)
+ #[allow(clippy::mut_from_ref)]
+ fn get_mut(&self, local_idx: LocalIdx) -> Option<&mut IggyPartition> {
+ // Safety: single-threaded per-shard model, no concurrent access.
+ unsafe { (&mut *self.partitions.get()).get_mut(*local_idx) }
}
/// Lookup local index by namespace.
@@ -134,7 +146,7 @@ impl<C> IggyPartitions<C> {
/// Insert a new partition and return its local index.
pub fn insert(&mut self, namespace: IggyNamespace, partition: IggyPartition) -> LocalIdx {
- let partitions = self.partitions_mut();
+ let partitions = self.partitions.get_mut();
let local_idx = LocalIdx::new(partitions.len());
partitions.push(partition);
self.namespace_to_local.insert(namespace, local_idx);
@@ -153,9 +165,11 @@ impl<C> IggyPartitions<C> {
}
/// Get mutable partition by namespace directly.
- pub fn get_mut_by_ns(&self, namespace: &IggyNamespace) -> Option<&mut IggyPartition> {
+ #[allow(clippy::mut_from_ref)]
+ fn get_mut_by_ns(&self, namespace: &IggyNamespace) -> Option<&mut IggyPartition> {
let idx = self.namespace_to_local.get(namespace)?;
- self.partitions_mut().get_mut(**idx)
+ // Safety: single-threaded per-shard model, no concurrent access.
+ unsafe { (&mut *self.partitions.get()).get_mut(**idx) }
}
/// Remove a partition by namespace. Returns the removed partition if found.
@@ -204,7 +218,8 @@ impl<C> IggyPartitions<C> {
/// Iterate over all (namespace, partition) pairs mutably.
pub fn iter_mut(&self) -> impl Iterator<Item = (&IggyNamespace, &mut IggyPartition)> {
- let partitions = self.partitions_mut();
+ // Safety: single-threaded per-shard model, no concurrent access.
+ let partitions = unsafe { &mut *self.partitions.get() };
let partitions_ptr = partitions.as_mut_ptr();
let partitions_len = partitions.len();
self.namespace_to_local.iter().filter_map(move |(ns, idx)| {
@@ -229,7 +244,7 @@ impl<C> IggyPartitions<C> {
self.insert(namespace, partition);
}
let idx = *self.namespace_to_local[&namespace];
- &mut self.partitions_mut()[idx]
+ &mut self.partitions.get_mut()[idx]
}
pub const fn consensus(&self) -> Option<&C> {
@@ -261,8 +276,10 @@ impl<C> IggyPartitions<C> {
// Create partition with initialized log
let stats = Arc::new(PartitionStats::default());
let mut partition = IggyPartition::new(stats);
- partition.log.add_persisted_segment(segment, storage);
- partition.offset.store(start_offset, Ordering::Relaxed);
+ partition
+ .log
+ .add_persisted_segment(segment, storage, None, None);
+ partition.offset.store(start_offset, Ordering::Release);
partition
.dirty_offset
.store(start_offset, Ordering::Relaxed);
@@ -285,11 +302,15 @@ impl<C> IggyPartitions<C> {
/// Idempotent: subsequent calls for the same namespace are no-ops.
/// Consensus must be set separately via `set_consensus`.
///
- /// # Panics
- /// Panics if segment storage creation fails.
- pub async fn init_partition(&mut self, namespace: IggyNamespace) -> LocalIdx {
+ /// # Errors
+ ///
+ /// Returns an `IggyError` if the backing segment storage or writers cannot be created.
+ pub async fn init_partition(
+ &mut self,
+ namespace: IggyNamespace,
+ ) -> Result<LocalIdx, IggyError> {
if let Some(idx) = self.local_idx(&namespace) {
- return idx;
+ return Ok(idx);
}
// Create initial segment with storage
@@ -321,20 +342,50 @@ impl<C> IggyPartitions<C> {
false, // file_exists (new segment)
)
.await
- .expect("Failed to create segment storage");
+ .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?;
+ let messages_size_bytes = storage
+ .messages_writer
+ .as_ref()
+ .ok_or_else(|| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?
+ .size_counter();
+ let messages_writer = Rc::new(
+ MessagesWriter::new(
+ &messages_path,
+ messages_size_bytes,
+ self.config.enforce_fsync,
+ false,
+ )
+ .await
+ .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?,
+ );
+ let index_writer = Rc::new(
+ IggyIndexWriter::new(
+ &index_path,
+ Rc::new(std::sync::atomic::AtomicU64::new(0)),
+ self.config.enforce_fsync,
+ false,
+ )
+ .await
+ .map_err(|_| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))?,
+ );
// Create partition with initialized log
let stats = Arc::new(PartitionStats::default());
let mut partition = IggyPartition::new(stats);
- partition.log.add_persisted_segment(segment, storage);
- partition.offset.store(start_offset, Ordering::Relaxed);
+ partition.log.add_persisted_segment(
+ segment,
+ storage,
+ Some(messages_writer),
+ Some(index_writer),
+ );
+ partition.offset.store(start_offset, Ordering::Release);
partition
.dirty_offset
.store(start_offset, Ordering::Relaxed);
partition.should_increment_offset = false;
partition.stats.increment_segments_count(1);
- self.insert(namespace, partition)
+ Ok(self.insert(namespace, partition))
}
}
@@ -348,9 +399,39 @@ where
.consensus()
.expect("on_request: consensus not initialized");
- debug!(?namespace, "handling partition request");
+ emit_sim_event(
+ SimEventKind::ClientRequestReceived,
+ &RequestLogEvent {
+ replica: ReplicaLogContext::from_consensus(consensus, PlaneKind::Partitions),
+ client_id: message.header().client,
+ request_id: message.header().request,
+ operation: message.header().operation,
+ },
+ );
+ let message = if message.header().operation == Operation::SendMessages {
+ match convert_request_message(namespace, message) {
+ Ok(message) => message,
+ Err(error) => {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ namespace_raw = namespace.inner(),
+ operation = ?Operation::SendMessages,
+ error = %error,
+ "failed to convert send_messages request"
+ );
+ return;
+ }
+ }
+ } else {
+ message
+ };
let prepare = message.project(consensus);
- pipeline_prepare_common(consensus, prepare, |prepare| self.on_replicate(prepare)).await;
+ pipeline_prepare_common(consensus, PlaneKind::Partitions, prepare, |prepare| {
+ self.on_replicate(prepare)
+ })
+ .await;
}
async fn on_replicate(&self, message: <VsrConsensus<B> as Consensus>::Message<PrepareHeader>) {
@@ -364,8 +445,15 @@ where
Ok(current_op) => current_op,
Err(reason) => {
warn!(
- replica = consensus.replica(),
- "on_replicate: ignoring ({reason})"
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ namespace_raw = header.namespace,
+ operation = ?header.operation,
+ reason = reason.as_str(),
+ "ignoring prepare during replicate preflight"
);
return;
}
@@ -373,26 +461,50 @@ where
let is_old_prepare = fence_old_prepare_by_commit(consensus, &header);
if is_old_prepare {
- warn!("received old prepare, not replicating");
- } else {
- self.replicate(message.clone()).await;
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ commit = consensus.commit(),
+ namespace_raw = header.namespace,
+ operation = ?header.operation,
+ "received old prepare, skipping replication"
+ );
+ return;
}
- // TODO: Make those assertions be toggleable through an feature flag, so they can be used only by simulator/tests.
- debug_assert_eq!(header.op, current_op + 1);
- consensus.sequencer().set_sequence(header.op);
- consensus.set_last_prepare_checksum(header.checksum);
-
// TODO: Figure out the flow of the partition operations.
// In metadata layer we assume that when an `on_request` or `on_replicate` is called, it's called from correct shard.
// I think we need to do the same here, which means that the code from below is unfallable, the partition should always exist by now!
- self.apply_replicated_operation(&namespace, message).await;
+ let message = self.replicate(message).await;
+ if let Err(error) = self.apply_replicated_operation(&namespace, message).await {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ op = header.op,
+ namespace_raw = namespace.inner(),
+ operation = ?header.operation,
+ %error,
+ "failed to apply replicated partition operation"
+ );
+ return;
+ }
- self.send_prepare_ok(&header).await;
+ // TODO: Make those assertions be toggleable through a feature flag, so they can be used only by simulator/tests.
+ debug_assert_eq!(header.op, current_op + 1);
+ consensus.sequencer().set_sequence(header.op);
+ consensus.set_last_prepare_checksum(header.checksum);
+ emit_namespace_progress_event(
+ SimEventKind::NamespaceProgressUpdated,
+ &ReplicaLogContext::from_consensus(consensus, PlaneKind::Partitions),
+ header.op,
+ consensus.pipeline().borrow().len(),
+ );
- if consensus.is_follower() {
- self.commit_journal(namespace);
- }
+ self.send_prepare_ok(&header).await;
}
async fn on_ack(&self, message: <VsrConsensus<B> as Consensus>::Message<PrepareOkHeader>) {
@@ -400,22 +512,37 @@ where
let consensus = self.consensus().expect("on_ack: consensus not initialized");
if let Err(reason) = ack_preflight(consensus) {
- warn!("on_ack: ignoring ({reason})");
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ view = consensus.view(),
+ op = header.op,
+ reason = reason.as_str(),
+ "ignoring ack during preflight"
+ );
return;
}
{
let pipeline = consensus.pipeline().borrow();
if pipeline
- .message_by_op_and_checksum(header.op, header.prepare_checksum)
+ .entry_by_op_and_checksum(header.op, header.prepare_checksum)
.is_none()
{
- debug!("on_ack: prepare not in pipeline op={}", header.op);
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ op = header.op,
+ prepare_checksum = header.prepare_checksum,
+ "ack target prepare not in pipeline"
+ );
return;
}
}
- consensus.handle_prepare_ok(header);
+ consensus.handle_prepare_ok(PlaneKind::Partitions, header);
// SAFETY(IGGY-66): Per-namespace drain independent of global commit.
//
@@ -443,68 +570,18 @@ where
return;
}
- // Advance global commit for VSR protocol correctness
- {
- let pipeline = consensus.pipeline().borrow();
- let new_commit = pipeline.global_commit_frontier(consensus.commit());
- drop(pipeline);
- consensus.advance_commit_number(new_commit);
- }
-
- if let (Some(first), Some(last)) = (drained.first(), drained.last()) {
- debug!(
- "on_ack: draining committed ops=[{}..={}] count={}",
- first.header.op,
- last.header.op,
- drained.len()
- );
- }
-
- let mut committed_ns: HashSet<IggyNamespace> = HashSet::new();
-
- for PipelineEntry {
- header: prepare_header,
- ..
- } in drained
- {
- let entry_namespace = IggyNamespace::from_raw(prepare_header.namespace);
-
- match prepare_header.operation {
- Operation::SendMessages => {
- if committed_ns.insert(entry_namespace) {
- self.commit_messages(&entry_namespace).await;
- }
- debug!("on_ack: messages committed for op={}", prepare_header.op,);
- }
- Operation::StoreConsumerOffset => {
- // TODO: Commit consumer offset update.
- debug!(
- "on_ack: consumer offset committed for op={}",
- prepare_header.op
- );
- }
- _ => {
- warn!(
- "on_ack: unexpected operation {:?} for op={}",
- prepare_header.operation, prepare_header.op
- );
- }
- }
-
- let generic_reply =
- build_reply_message(consensus, &prepare_header, bytes::Bytes::new()).into_generic();
- debug!(
- "on_ack: sending reply to client={} for op={}",
- prepare_header.client, prepare_header.op
- );
-
- // TODO: Propagate send error instead of panicking; requires bus error design.
- consensus
- .message_bus()
- .send_to_client(prepare_header.client, generic_reply)
- .await
- .unwrap();
- }
+ self.handle_committed_entries(consensus, drained).await;
+
+ let pipeline = consensus.pipeline().borrow();
+ let new_commit = pipeline.global_commit_frontier(consensus.commit());
+ drop(pipeline);
+ consensus.advance_commit_number(new_commit);
+ emit_namespace_progress_event(
+ SimEventKind::NamespaceProgressUpdated,
+ &ReplicaLogContext::from_consensus(consensus, PlaneKind::Partitions),
+ new_commit,
+ consensus.pipeline().borrow().len(),
+ );
}
}
@@ -542,7 +619,7 @@ where
&self,
namespace: &IggyNamespace,
message: Message<PrepareHeader>,
- ) {
+ ) -> Result<(), IggyError> {
let consensus = self
.consensus()
.expect("apply_replicated_operation: consensus not initialized");
@@ -553,31 +630,54 @@ where
match header.operation {
Operation::SendMessages => {
self.append_send_messages_to_journal(namespace, message)
- .await;
+ .await?;
debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
replica = consensus.replica(),
op = header.op,
- ?namespace,
- "on_replicate: send_messages appended to partition journal"
+ namespace_raw = namespace.inner(),
+ operation = ?header.operation,
+ "replicated send_messages appended to partition journal"
);
+ Ok(())
}
Operation::StoreConsumerOffset => {
- let body = message.body_bytes();
- let body = body.as_ref();
- let consumer_kind = body[0];
- let consumer_id = u32::from_le_bytes(body[1..5].try_into().unwrap()) as usize;
- let offset = u64::from_le_bytes(body[5..13].try_into().unwrap());
+ let total_size = header.size() as usize;
+ let body = &message.as_slice()[std::mem::size_of::<PrepareHeader>()..total_size];
+ let consumer_kind = *body.first().ok_or(IggyError::InvalidCommand)?;
+ let consumer_id =
+ body.get(1..5)
+ .ok_or(IggyError::InvalidCommand)
+ .and_then(|bytes| {
+ <[u8; 4]>::try_from(bytes)
+ .map(u32::from_le_bytes)
+ .map(|value| value as usize)
+ .map_err(|_| IggyError::InvalidCommand)
+ })?;
+ let offset =
+ body.get(5..13)
+ .ok_or(IggyError::InvalidCommand)
+ .and_then(|bytes| {
+ <[u8; 8]>::try_from(bytes)
+ .map(u64::from_le_bytes)
+ .map_err(|_| IggyError::InvalidCommand)
+ })?;
let consumer = match consumer_kind {
1 => PollingConsumer::Consumer(consumer_id, 0),
2 => PollingConsumer::ConsumerGroup(consumer_id, 0),
_ => {
warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
replica = consensus.replica(),
op = header.op,
+ namespace_raw = namespace.inner(),
+ operation = ?header.operation,
consumer_kind,
- "on_replicate: unknown consumer kind"
+ "unknown consumer kind while applying replicated offset update"
);
- return;
+ return Err(IggyError::InvalidCommand);
}
};
@@ -587,21 +687,30 @@ where
let _ = partition.store_consumer_offset(consumer, offset);
debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
replica = consensus.replica(),
op = header.op,
+ namespace_raw = namespace.inner(),
+ operation = ?header.operation,
consumer_kind,
consumer_id,
offset,
- "on_replicate: consumer offset stored"
+ "replicated consumer offset stored"
);
+ Ok(())
}
_ => {
warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
replica = consensus.replica(),
+ namespace_raw = namespace.inner(),
op = header.op,
- "on_replicate: unexpected operation {:?}",
- header.operation
+ operation = ?header.operation,
+ "unexpected replicated partition operation"
);
+ Ok(())
}
}
}
@@ -618,36 +727,43 @@ where
&self,
namespace: &IggyNamespace,
message: Message<PrepareHeader>,
- ) {
+ ) -> Result<(), IggyError> {
+ let write_lock = self
+ .get_by_ns(namespace)
+ .expect("append_send_messages_to_journal: partition not found for namespace")
+ .write_lock
+ .clone();
+ let _guard = write_lock.lock().await;
let partition = self
.get_mut_by_ns(namespace)
.expect("append_send_messages_to_journal: partition not found for namespace");
- let _ = partition.append_messages(message).await;
+ partition.append_messages(message).await.map(|_| ())
}
/// Replicate a prepare message to the next replica in the chain.
///
/// Chain replication: primary -> first backup -> ... -> last backup.
/// Stops when the next replica would be the primary.
- async fn replicate(&self, message: Message<PrepareHeader>) {
+ async fn replicate(&self, message: Message<PrepareHeader>) -> Message<PrepareHeader> {
let consensus = self
.consensus()
.expect("replicate: consensus not initialized");
- replicate_to_next_in_chain(consensus, message).await;
- }
-
- #[allow(clippy::unused_self, clippy::missing_const_for_fn)]
- fn commit_journal(&self, _namespace: IggyNamespace) {
- // TODO: Implement commit logic for followers.
- // Walk through journal from last committed to current commit number
- // Apply each entry to the partition state
+ replicate_to_next_in_chain(consensus, message).await
}
- /// Commit messages after quorum ack.
+ /// Persist prepared messages to segment storage and advance the durable frontier.
///
/// Updates segment metadata, stats, flushes journal to disk if thresholds
- /// are exceeded, and advances the committed offset last.
- async fn commit_messages(&self, namespace: &IggyNamespace) {
+ /// are exceeded, and advances the durable offset last.
+ #[allow(clippy::too_many_lines)]
+ async fn commit_messages(&self, namespace: &IggyNamespace) -> Result<(), IggyError> {
+ let write_lock = self
+ .get_by_ns(namespace)
+ .expect("commit_messages: partition not found for namespace")
+ .write_lock
+ .clone();
+ let _guard = write_lock.lock().await;
+
let partition = self
.get_mut_by_ns(namespace)
.expect("commit_messages: partition not found for namespace");
@@ -656,11 +772,92 @@ where
let journal_info = journal.info;
if journal_info.messages_count == 0 {
- return;
+ return Ok(());
+ }
+
+ // 1. Check flush thresholds.
+ let is_full = partition.log.active_segment().is_full();
+
+ let unsaved_messages_count_exceeded =
+ journal_info.messages_count >= self.config.messages_required_to_save;
+ let unsaved_messages_size_exceeded = journal_info.size.as_bytes_u64()
+ >= self.config.size_of_messages_required_to_save.as_bytes_u64();
+ let should_persist =
+ is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded;
+
+ if !should_persist {
+ return Ok(());
+ }
+
+ // Freeze journal batches.
+ let (frozen_batches, index_bytes, batch_count) = {
+ let entries = partition.log.journal().inner.entries();
+ let segment = partition.log.active_segment();
+ let mut file_position = segment.size.as_bytes_u64();
+ partition.log.ensure_indexes();
+ let indexes = partition.log.active_indexes_mut().unwrap();
+ let mut flush_index = None;
+ let mut frozen = Vec::with_capacity(entries.len());
+ let mut batch_count = 0u32;
+
+ for entry in entries {
+ let Ok(batch) = decode_prepare_slice(entry.as_slice()) else {
+ continue;
+ };
+ let message_count = batch.message_count();
+ if message_count == 0 {
+ continue;
+ }
+
+ let index = crate::iggy_index::IggyIndex::new(
+ batch.header.base_offset,
+ batch.header.base_timestamp,
+ file_position,
+ );
+ if flush_index.is_none() {
+ indexes.insert(index.offset, index.timestamp, index.position);
+ flush_index = Some(index);
+ }
+ file_position += batch.header.total_size() as u64;
+ batch_count += message_count;
+ frozen.push(entry);
+ }
+
+ let index_bytes =
+ flush_index.map(|index| crate::iggy_index::IggyIndexCache::serialize(&index));
+
+ (frozen, index_bytes, batch_count)
+ };
+
+ let Some(index_bytes) = index_bytes else {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ "failed to build sparse index entry from pending journal batches"
+ );
+ return Err(IggyError::InvalidCommand);
+ };
+
+ // Persist to disk.
+ self.persist_frozen_batches_to_disk(namespace, frozen_batches, index_bytes, batch_count)
+ .await?;
+
+ if is_full {
+ self.rotate_segment(namespace).await?;
}
- // 1. Update segment metadata from journal state.
- // Note: segment.current_position is already updated in append_batch (prepare phase).
+ // Reset journal info after drain.
+ let partition = self
+ .get_mut_by_ns(namespace)
+ .expect("commit_messages: partition not found");
+ let _ = partition.log.journal_mut().inner.commit();
+ partition.log.journal_mut().info = JournalInfo::default();
+
+ // Update segment metadata from the just-persisted journal state.
+ let partition = self
+ .get_mut_by_ns(namespace)
+ .expect("commit_messages: partition not found");
let segment_index = partition.log.segments().len() - 1;
let segment = &mut partition.log.segments_mut()[segment_index];
@@ -668,9 +865,9 @@ where
segment.start_timestamp = journal_info.first_timestamp;
}
segment.end_timestamp = journal_info.end_timestamp;
+ segment.max_timestamp = segment.max_timestamp.max(journal_info.max_timestamp);
segment.end_offset = journal_info.current_offset;
- // 2. Update stats.
partition
.stats
.increment_size_bytes(journal_info.size.as_bytes_u64());
@@ -678,67 +875,246 @@ where
.stats
.increment_messages_count(u64::from(journal_info.messages_count));
- // 3. Check flush thresholds.
- let is_full = segment.is_full();
+ let durable_offset = journal_info.current_offset;
+ partition.offset.store(durable_offset, Ordering::Release);
+ partition.stats.set_current_offset(durable_offset);
+ Ok(())
+ }
- let unsaved_messages_count_exceeded =
- journal_info.messages_count >= self.config.messages_required_to_save;
- let unsaved_messages_size_exceeded = journal_info.size.as_bytes_u64()
- >= self.config.size_of_messages_required_to_save.as_bytes_u64();
+ async fn handle_committed_entries(
+ &self,
+ consensus: &VsrConsensus<B, NamespacedPipeline>,
+ drained: Vec<PipelineEntry>,
+ ) {
+ if let (Some(first), Some(last)) = (drained.first(), drained.last()) {
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ first_op = first.header.op,
+ last_op = last.header.op,
+ drained_count = drained.len(),
+ "draining committed partition ops"
+ );
+ }
+
+ let mut committed_ns: HashSet<IggyNamespace> = HashSet::new();
+ let mut failed_ns: HashSet<IggyNamespace> = HashSet::new();
+ let committed_visible_offsets = self
+ .resolve_committed_visible_offsets(consensus, &drained, &mut failed_ns)
+ .await;
+
+ for PipelineEntry {
+ header: prepare_header,
+ ..
+ } in drained
+ {
+ let entry_namespace = IggyNamespace::from_raw(prepare_header.namespace);
+ if !self
+ .commit_partition_entry(
+ consensus,
+ prepare_header,
+ entry_namespace,
+ &mut committed_ns,
+ &committed_visible_offsets,
+ &mut failed_ns,
+ )
+ .await
+ {
+ continue;
+ }
- if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded {
- // Freeze journal batches.
- let frozen_batches = {
- let batches = partition.log.journal_mut().inner.commit();
- partition.log.ensure_indexes();
- batches.append_indexes_to(partition.log.active_indexes_mut().unwrap());
-
- let frozen: Vec<_> = batches
- .into_inner()
- .into_iter()
- .map(|mut b| b.freeze())
- .collect();
- partition.log.set_in_flight(frozen.clone());
- frozen
+ let pipeline_depth = consensus.pipeline().borrow().len();
+ let event = CommitLogEvent {
+ replica: ReplicaLogContext::from_consensus(consensus, PlaneKind::Partitions),
+ op: prepare_header.op,
+ client_id: prepare_header.client,
+ request_id: prepare_header.request,
+ operation: prepare_header.operation,
+ pipeline_depth,
};
+ emit_sim_event(SimEventKind::OperationCommitted, &event);
+ emit_namespace_progress_event(
+ SimEventKind::NamespaceProgressUpdated,
+ &event.replica,
+ prepare_header.op,
+ pipeline_depth,
+ );
- // Persist to disk.
- self.persist_frozen_batches_to_disk(namespace, frozen_batches)
- .await;
+ let generic_reply =
+ build_reply_message(consensus, &prepare_header, bytes::Bytes::new()).into_generic();
+ let reply_buffers = freeze_client_reply(generic_reply);
+ emit_sim_event(SimEventKind::ClientReplyEmitted, &event);
+
+ if let Err(error) = consensus
+ .message_bus()
+ .send_to_client(prepare_header.client, reply_buffers)
+ .await
+ {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ client = prepare_header.client,
+ op = prepare_header.op,
+ namespace_raw = entry_namespace.inner(),
+ %error,
+ "failed to send reply to client"
+ );
+ }
+ }
+ if !failed_ns.is_empty() {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ failed_namespaces = failed_ns.len(),
+ "some namespaces failed local commit handling"
+ );
+ }
+ }
- if is_full {
- self.rotate_segment(namespace).await;
+ async fn resolve_committed_visible_offsets(
+ &self,
+ consensus: &VsrConsensus<B, NamespacedPipeline>,
+ drained: &[PipelineEntry],
+ failed_ns: &mut HashSet<IggyNamespace>,
+ ) -> HashMap<IggyNamespace, u64> {
+ let mut committed_visible_offsets = HashMap::new();
+
+ for entry in drained {
+ if entry.header.operation != Operation::SendMessages {
+ continue;
}
- // Reset journal info after drain.
- let partition = self
- .get_mut_by_ns(namespace)
- .expect("commit_messages: partition not found");
- partition.log.journal_mut().info = JournalInfo::default();
+ let entry_namespace = IggyNamespace::from_raw(entry.header.namespace);
+ match self
+ .committed_end_offset_for_prepare(&entry_namespace, &entry.header)
+ .await
+ {
+ Ok(Some(end_offset)) => {
+ committed_visible_offsets.insert(entry_namespace, end_offset);
+ }
+ Ok(None) => {}
+ Err(error) => {
+ failed_ns.insert(entry_namespace);
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ namespace_raw = entry_namespace.inner(),
+ op = entry.header.op,
+ operation = ?entry.header.operation,
+ %error,
+ "failed to resolve committed visible offset for partition entry"
+ );
+ }
+ }
+ }
+
+ committed_visible_offsets
+ }
+
+ async fn commit_partition_entry(
+ &self,
+ consensus: &VsrConsensus<B, NamespacedPipeline>,
+ prepare_header: PrepareHeader,
+ entry_namespace: IggyNamespace,
+ committed_ns: &mut HashSet<IggyNamespace>,
+ committed_visible_offsets: &HashMap<IggyNamespace, u64>,
+ failed_ns: &mut HashSet<IggyNamespace>,
+ ) -> bool {
+ match prepare_header.operation {
+ Operation::SendMessages => {
+ if committed_ns.insert(entry_namespace)
+ && let Err(error) = self.commit_messages(&entry_namespace).await
+ {
+ failed_ns.insert(entry_namespace);
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ namespace_raw = entry_namespace.inner(),
+ op = prepare_header.op,
+ operation = ?prepare_header.operation,
+ %error,
+ "failed to commit partition messages"
+ );
+ }
+
+ if committed_ns.contains(&entry_namespace)
+ && !failed_ns.contains(&entry_namespace)
+ && let Some(visible_offset) =
+ committed_visible_offsets.get(&entry_namespace).copied()
+ {
+ let partition = self
+ .get_by_ns(&entry_namespace)
+ .expect("commit_partition_entry: partition not found");
+ partition.offset.store(visible_offset, Ordering::Release);
+ partition.stats.set_current_offset(visible_offset);
+ }
+
+ !failed_ns.contains(&entry_namespace)
+ }
+ Operation::StoreConsumerOffset => {
+ // TODO: Commit consumer offset update.
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ op = prepare_header.op,
+ namespace_raw = entry_namespace.inner(),
+ "consumer offset committed"
+ );
+ true
+ }
+ _ => {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ replica_id = consensus.replica(),
+ op = prepare_header.op,
+ namespace_raw = entry_namespace.inner(),
+ operation = ?prepare_header.operation,
+ "unexpected committed partition operation"
+ );
+ true
+ }
}
+ }
- // 4. Advance committed offset (last, so consumers only see offset after data is durable).
+ async fn committed_end_offset_for_prepare(
+ &self,
+ namespace: &IggyNamespace,
+ prepare_header: &PrepareHeader,
+ ) -> Result<Option<u64>, IggyError> {
let partition = self
- .get_mut_by_ns(namespace)
- .expect("commit_messages: partition not found");
- let committed_offset = journal_info.current_offset;
- partition.offset.store(committed_offset, Ordering::Relaxed);
- partition.stats.set_current_offset(committed_offset);
+ .get_by_ns(namespace)
+ .expect("committed_end_offset_for_prepare: partition not found");
+ let Some(entry) = partition.log.journal().inner.entry(prepare_header).await else {
+ return Err(IggyError::InvalidCommand);
+ };
+ let batch =
+ decode_prepare_slice(entry.as_slice()).map_err(|_| IggyError::InvalidCommand)?;
+ let message_count = batch.message_count();
+ if message_count == 0 {
+ return Ok(None);
+ }
+
+ Ok(Some(
+ batch.header.base_offset + u64::from(message_count) - 1,
+ ))
}
/// Persist frozen batches to disk and update segment bookkeeping.
async fn persist_frozen_batches_to_disk(
&self,
namespace: &IggyNamespace,
- frozen_batches: Vec<iggy_common::IggyMessagesBatch>,
- ) {
- let batch_count: u32 = frozen_batches
- .iter()
- .map(iggy_common::IggyMessagesBatch::count)
- .sum();
-
+ frozen_batches: Vec<Frozen<4096>>,
+ index_bytes: Vec<u8>,
+ batch_count: u32,
+ ) -> Result<(), IggyError> {
if batch_count == 0 {
- return;
+ return Ok(());
}
let partition = self
@@ -746,48 +1122,88 @@ where
.expect("persist: partition not found");
if !partition.log.has_segments() {
- return;
+ return Ok(());
}
+ let stripped_batches: Vec<_> = frozen_batches
+ .into_iter()
+ .map(|batch| batch.slice(std::mem::size_of::<PrepareHeader>()..))
+ .collect();
let messages_writer = partition
.log
- .active_storage()
- .messages_writer
- .as_ref()
- .expect("Messages writer not initialized")
- .clone();
+ .messages_writers()
+ .last()
+ .and_then(|writer| writer.as_ref())
+ .cloned();
let index_writer = partition
.log
- .active_storage()
- .index_writer
- .as_ref()
- .expect("Index writer not initialized")
- .clone();
+ .index_writers()
+ .last()
+ .and_then(|writer| writer.as_ref())
+ .cloned();
- let saved = messages_writer
- .as_ref()
- .save_frozen_batches(&frozen_batches)
- .await
- .expect("persist: failed to save frozen batches");
+ if messages_writer.is_none() || index_writer.is_none() {
+ let saved_bytes = stripped_batches.iter().map(Frozen::len).sum::<usize>();
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ batch_count,
+ saved_bytes,
+ "simulated in-memory batch persistence"
+ );
- let unsaved_indexes_slice = {
let partition = self
- .get_by_ns(namespace)
+ .get_mut_by_ns(namespace)
.expect("persist: partition not found");
let segment_index = partition.log.segments().len() - 1;
- partition.log.indexes()[segment_index]
- .as_ref()
- .expect("indexes must exist for segment being persisted")
- .unsaved_slice()
- };
+ let segment = &mut partition.log.segments_mut()[segment_index];
+ segment.size = IggyByteSize::from(segment.size.as_bytes_u64() + saved_bytes as u64);
+ partition.log.clear_in_flight();
+ return Ok(());
+ }
- index_writer
- .as_ref()
- .save_indexes(unsaved_indexes_slice)
+ let messages_writer = messages_writer.expect("checked above");
+ let index_writer = index_writer.expect("checked above");
+
+ let saved = messages_writer
+ .save_frozen_batches(&stripped_batches)
.await
- .expect("persist: failed to save indexes");
+ .map_err(|error| {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ batch_count,
+ %error,
+ "failed to save frozen batches"
+ );
+ error
+ })?;
- debug!(?namespace, batch_count, ?saved, "persisted batches to disk");
+ index_writer
+ .save_indexes(index_bytes)
+ .await
+ .map_err(|error| {
+ warn!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ batch_count,
+ %error,
+ "failed to save sparse indexes"
+ );
+ error
+ })?;
+
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ batch_count,
+ saved_bytes = saved.as_bytes_u64(),
+ "persisted batches to disk"
+ );
let partition = self
.get_mut_by_ns(namespace)
@@ -796,19 +1212,15 @@ where
// Recalculate index: segment deletion during async I/O shifts indices.
let segment_index = partition.log.segments().len() - 1;
- let indexes = partition.log.indexes_mut()[segment_index]
- .as_mut()
- .expect("indexes must exist for segment being persisted");
- indexes.mark_saved();
-
let segment = &mut partition.log.segments_mut()[segment_index];
segment.size = IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64());
partition.log.clear_in_flight();
+ Ok(())
}
/// Rotate to a new segment when the current one is full.
- async fn rotate_segment(&self, namespace: &IggyNamespace) {
+ async fn rotate_segment(&self, namespace: &IggyNamespace) -> Result<(), IggyError> {
let partition = self
.get_mut_by_ns(namespace)
.expect("rotate_segment: partition not found");
@@ -845,33 +1257,61 @@ where
false, // file_exists (new segment)
)
.await
- .expect("Failed to create segment storage");
-
- // Clear old segment's indexes.
- // TODO: Waiting for issue to move server config to shared module.
- // Once complete, conditionally clear based on cache_indexes config:
- // if !matches!(self.config.cache_indexes, CacheIndexesConfig::All) {
- // partition.log.indexes_mut()[old_segment_index] = None;
- // }
- partition.log.indexes_mut()[old_segment_index] = None;
+ .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?;
+ let messages_size_bytes = storage
+ .messages_writer
+ .as_ref()
+ .ok_or_else(|| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?
+ .size_counter();
+ let messages_writer = Rc::new(
+ MessagesWriter::new(
+ &messages_path,
+ messages_size_bytes,
+ self.config.enforce_fsync,
+ false,
+ )
+ .await
+ .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?,
+ );
+ let index_writer = Rc::new(
+ IggyIndexWriter::new(
+ &index_path,
+ Rc::new(std::sync::atomic::AtomicU64::new(0)),
+ self.config.enforce_fsync,
+ false,
+ )
+ .await
+ .map_err(|_| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))?,
+ );
// Close writers for the sealed segment.
let old_storage = &mut partition.log.storages_mut()[old_segment_index];
let _ = old_storage.shutdown();
-
- partition.log.add_persisted_segment(segment, storage);
+ partition.log.messages_writers_mut()[old_segment_index] = None;
+ partition.log.index_writers_mut()[old_segment_index] = None;
+
+ partition.log.add_persisted_segment(
+ segment,
+ storage,
+ Some(messages_writer),
+ Some(index_writer),
+ );
partition.stats.increment_segments_count(1);
- debug!(?namespace, start_offset, "rotated to new segment");
+ debug!(
+ target: "iggy.partitions.diag",
+ plane = "partitions",
+ namespace_raw = namespace.inner(),
+ start_offset,
+ "rotated to new segment"
+ );
+ Ok(())
}
async fn send_prepare_ok(&self, header: &PrepareHeader) {
let consensus = self
.consensus()
.expect("send_prepare_ok: consensus not initialized");
- // TODO: Verify the prepare is persisted in the partition journal.
- // The partition journal uses MessageLookup headers, so we cannot
- // check by PrepareHeader.op directly. For now, skip this check.
- send_prepare_ok_common(consensus, header, None).await;
+ send_prepare_ok_common(consensus, header, Some(true)).await;
}
}
diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs
index c21cc8d60..dc020b492 100644
--- a/core/partitions/src/journal.rs
+++ b/core/partitions/src/journal.rs
@@ -15,32 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-use bytes::Bytes;
-use iggy_binary_protocol::{Message, Operation, PrepareHeader};
-use iggy_common::{IggyMessagesBatchMut, IggyMessagesBatchSet};
+use iggy_binary_protocol::{Operation, PrepareHeader};
+use iggy_common::send_messages2::{COMMAND_HEADER_SIZE, SendMessages2Ref, decode_prepare_slice};
+use iobuf::{Frozen, Owned};
use journal::{Journal, Storage};
+use std::io;
use std::{
cell::UnsafeCell,
collections::{BTreeMap, HashMap},
};
-// TODO: Fix that, we need to figure out how to store the `IggyMessagesBatchSet`.
-/// No-op storage backend for the in-memory partition journal.
-#[allow(dead_code)]
-#[derive(Debug)]
-pub struct Noop;
-
-impl Storage for Noop {
- type Buffer = ();
+use crate::{Fragment, PollFragments, PollQueryResult};
- async fn write_at(&self, _offset: usize, _buf: ()) -> std::io::Result<usize> {
- Ok(0)
- }
-
- async fn read_at(&self, _offset: usize, _buffer: ()) -> std::io::Result<()> {
- Ok(())
- }
-}
+const ZERO_LEN: usize = 0;
+type JournalBuffer = Frozen<4096>;
/// Lookup key for querying messages from the journal.
#[derive(Debug, Clone, Copy)]
@@ -51,14 +39,22 @@ pub enum MessageLookup {
Timestamp { timestamp: u64, count: u32 },
}
-impl std::ops::Deref for MessageLookup {
- type Target = Self;
-
- fn deref(&self) -> &Self {
- self
+impl MessageLookup {
+ const fn count(self) -> u32 {
+ match self {
+ Self::Offset { count, .. } | Self::Timestamp { count, .. } => count,
+ }
}
}
+#[derive(Debug, Clone, Copy)]
+struct SelectedBatchSlice {
+ start: usize,
+ end: usize,
+ matched_messages: u32,
+ last_matching_offset: u64,
+}
+
#[allow(dead_code)]
pub trait QueryableJournal<S>: Journal<S>
where
@@ -66,12 +62,12 @@ where
{
type Query;
- fn get(&self, query: &Self::Query) -> impl Future<Output = Option<IggyMessagesBatchSet>>;
+ fn get(&self, query: &Self::Query) -> impl Future<Output = Option<PollQueryResult<4096>>>;
}
#[derive(Debug, Default)]
pub struct PartitionJournalMemStorage {
- entries: UnsafeCell<Vec<Bytes>>,
+ entries: UnsafeCell<Vec<JournalBuffer>>,
/// Maps byte offset (as if disk-backed) to index in entries Vec
offset_to_index: UnsafeCell<HashMap<usize, usize>>,
/// Current write position (cumulative byte offset)
@@ -79,9 +75,9 @@ pub struct PartitionJournalMemStorage {
}
impl Storage for PartitionJournalMemStorage {
- type Buffer = Bytes;
+ type Buffer = JournalBuffer;
- async fn write_at(&self, _offset: usize, buf: Self::Buffer) -> std::io::Result<usize> {
+ async fn write_at(&self, _offset: usize, buf: Self::Buffer) -> io::Result<usize> {
let len = buf.len();
let entries = unsafe { &mut *self.entries.get() };
let offset_to_index = unsafe { &mut *self.offset_to_index.get() };
@@ -90,41 +86,45 @@ impl Storage for PartitionJournalMemStorage {
let index = entries.len();
offset_to_index.insert(*current_offset, index);
entries.push(buf);
-
- let write_offset = *current_offset;
*current_offset += len;
- Ok(write_offset)
+ Ok(len)
}
- async fn read_at(&self, offset: usize, _buffer: Self::Buffer) -> std::io::Result<Self::Buffer> {
+ async fn read_at(&self, offset: usize, _buffer: Self::Buffer) -> io::Result<Self::Buffer> {
let offset_to_index = unsafe { &*self.offset_to_index.get() };
let Some(&index) = offset_to_index.get(&offset) else {
- return Ok(Bytes::new());
+ return Ok(Owned::<4096>::zeroed(0).into());
};
let entries = unsafe { &*self.entries.get() };
- Ok(entries.get(index).cloned().unwrap_or_default())
+ Ok(entries
+ .get(index)
+ .cloned()
+ .unwrap_or_else(|| Owned::<4096>::zeroed(0).into()))
}
}
pub struct PartitionJournal<S>
where
- S: Storage<Buffer = Bytes>,
+ S: Storage<Buffer = JournalBuffer>,
{
/// Maps op -> storage byte offset (for all entries)
op_to_storage_offset: UnsafeCell<BTreeMap<u64, usize>>,
/// Maps message offset -> op (for queryable entries)
offset_to_op: UnsafeCell<BTreeMap<u64, u64>>,
- /// Maps timestamp -> op (for queryable entries)
- timestamp_to_op: UnsafeCell<BTreeMap<u64, u64>>,
+ /// Maps `(origin_timestamp, op)` -> op (for queryable entries).
+ ///
+ /// Keeping `op` in the key preserves duplicate timestamps while still
+ /// letting us seek to the closest batch for timestamp-based polling.
+ timestamp_to_op: UnsafeCell<BTreeMap<(u64, u64), u64>>,
headers: UnsafeCell<Vec<PrepareHeader>>,
inner: UnsafeCell<JournalInner<S>>,
}
impl<S> Default for PartitionJournal<S>
where
- S: Storage<Buffer = Bytes> + Default,
+ S: Storage<Buffer = JournalBuffer> + Default,
{
fn default() -> Self {
Self {
@@ -141,7 +141,7 @@ where
impl<S> std::fmt::Debug for PartitionJournal<S>
where
- S: Storage<Buffer = Bytes>,
+ S: Storage<Buffer = JournalBuffer>,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PartitionJournal2Impl").finish()
@@ -150,13 +150,18 @@ where
struct JournalInner<S>
where
- S: Storage<Buffer = Bytes>,
+ S: Storage<Buffer = JournalBuffer>,
{
storage: S,
}
impl PartitionJournalMemStorage {
- fn drain(&self) -> Vec<Bytes> {
+ fn entries(&self) -> Vec<JournalBuffer> {
+ let entries = unsafe { &*self.entries.get() };
+ entries.clone()
+ }
+
+ fn drain(&self) -> Vec<JournalBuffer> {
let entries = unsafe { &mut *self.entries.get() };
let offset_to_index = unsafe { &mut *self.offset_to_index.get() };
let current_offset = unsafe { &mut *self.current_offset.get() };
@@ -171,23 +176,26 @@ impl PartitionJournalMemStorage {
let entries = unsafe { &*self.entries.get() };
entries.is_empty()
}
+
+ fn current_offset(&self) -> usize {
+ let current_offset = unsafe { &*self.current_offset.get() };
+ *current_offset
+ }
}
impl PartitionJournal<PartitionJournalMemStorage> {
+ pub fn entries(&self) -> Vec<JournalBuffer> {
+ let inner = unsafe { &*self.inner.get() };
+ inner.storage.entries()
+ }
+
/// Drain all accumulated batches, matching the legacy `PartitionJournal` API.
- pub fn commit(&self) -> IggyMessagesBatchSet {
+ pub fn commit(&self) -> Vec<JournalBuffer> {
let entries = {
let inner = unsafe { &*self.inner.get() };
inner.storage.drain()
};
- let mut messages = Vec::with_capacity(entries.len());
- for bytes in entries {
- if let Ok(message) = Message::from_bytes(bytes) {
- messages.push(message);
- }
- }
-
let headers = unsafe { &mut *self.headers.get() };
headers.clear();
let op_to_storage_offset = unsafe { &mut *self.op_to_storage_offset.get() };
@@ -197,7 +205,7 @@ impl PartitionJournal<PartitionJournalMemStorage> {
let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() };
timestamp_to_op.clear();
- Self::messages_to_batch_set(&messages)
+ entries
}
pub fn is_empty(&self) -> bool {
@@ -208,26 +216,17 @@ impl PartitionJournal<PartitionJournalMemStorage> {
impl<S> PartitionJournal<S>
where
- S: Storage<Buffer = Bytes>,
+ S: Storage<Buffer = JournalBuffer>,
{
- fn message_to_batch(message: &Message<PrepareHeader>) -> Option<IggyMessagesBatchMut> {
- if message.header().operation != Operation::SendMessages {
- return None;
- }
-
- crate::decode_send_messages_batch(message.body_bytes())
- }
-
- fn messages_to_batch_set(messages: &[Message<PrepareHeader>]) -> IggyMessagesBatchSet {
- let mut batch_set = IggyMessagesBatchSet::empty();
-
- for message in messages {
- if let Some(batch) = Self::message_to_batch(message) {
- batch_set.add_batch(batch);
- }
+ #[must_use]
+ pub const fn with_storage(storage: S) -> Self {
+ Self {
+ op_to_storage_offset: UnsafeCell::new(BTreeMap::new()),
+ offset_to_op: UnsafeCell::new(BTreeMap::new()),
+ timestamp_to_op: UnsafeCell::new(BTreeMap::new()),
+ headers: UnsafeCell::new(Vec::new()),
+ inner: UnsafeCell::new(JournalInner { storage }),
}
-
- batch_set
}
#[allow(dead_code)]
@@ -243,16 +242,27 @@ where
}
MessageLookup::Timestamp { timestamp, .. } => {
let timestamp_to_op = unsafe { &*self.timestamp_to_op.get() };
+ let next_at_or_after = timestamp_to_op
+ .range((*timestamp, 0)..)
+ .next()
+ .map(|(key, op)| (*key, *op));
+
+ if let Some(((candidate_timestamp, _), op)) = next_at_or_after
+ && candidate_timestamp == *timestamp
+ {
+ return Some(op);
+ }
+
timestamp_to_op
- .range(..=*timestamp)
+ .range(..(*timestamp, 0))
.next_back()
- .or_else(|| timestamp_to_op.range(*timestamp..).next())
.map(|(_, op)| *op)
+ .or_else(|| next_at_or_after.map(|(_, op)| op))
}
}
}
- async fn message_by_op(&self, op: u64) -> Option<Message<PrepareHeader>> {
+ async fn bytes_by_op(&self, op: u64) -> Option<JournalBuffer> {
let storage_offset = {
let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() };
*op_to_storage_offset.get(&op)?
@@ -262,29 +272,28 @@ where
let inner = unsafe { &*self.inner.get() };
inner
.storage
- .read_at(storage_offset, Bytes::new())
+ .read_at(storage_offset, Owned::<4096>::zeroed(ZERO_LEN).into())
.await
- .unwrap_or_default()
+ .unwrap_or_else(|_| Owned::<4096>::zeroed(ZERO_LEN).into())
};
if bytes.is_empty() {
return None;
}
- Some(
- Message::from_bytes(bytes)
- .expect("partition.journal.storage.read: invalid bytes for message"),
- )
+ Some(bytes)
}
#[allow(dead_code)]
- async fn load_messages_from_storage(
+ async fn load_polled_batches_from_storage(
&self,
start_op: u64,
- count: u32,
- ) -> Vec<Message<PrepareHeader>> {
+ query: MessageLookup,
+ ) -> PollQueryResult<4096> {
+ let count = query.count();
+
if count == 0 {
- return Vec::new();
+ return (PollFragments::new(), None);
}
// Get (op, storage_offset) pairs directly from the mapping
@@ -297,11 +306,12 @@ where
.collect()
};
- let mut messages = Vec::new();
- let mut loaded_messages = 0u32;
+ let mut fragments = PollFragments::new();
+ let mut last_matching_offset = None;
+ let mut matched_messages = 0u32;
for (_, storage_offset) in op_offsets {
- if loaded_messages >= count {
+ if matched_messages >= count {
break;
}
@@ -309,36 +319,49 @@ where
let inner = unsafe { &*self.inner.get() };
inner
.storage
- .read_at(storage_offset, Bytes::new())
+ .read_at(storage_offset, Owned::<4096>::zeroed(ZERO_LEN).into())
.await
- .unwrap_or_default()
+ .unwrap_or_else(|_| Owned::<4096>::zeroed(ZERO_LEN).into())
};
if bytes.is_empty() {
continue;
}
- let message = Message::from_bytes(bytes)
- .expect("partition.journal.storage.read: invalid bytes for message");
-
- if let Some(batch) = Self::message_to_batch(&message) {
- loaded_messages = loaded_messages.saturating_add(batch.count());
- messages.push(message);
+ let header_size = std::mem::size_of::<PrepareHeader>();
+ let header_bytes = &bytes[..header_size];
+ let header = *bytemuck::checked::try_from_bytes::<PrepareHeader>(header_bytes)
+ .expect("partition journal storage must contain a valid prepare header");
+ if header.operation != Operation::SendMessages {
+ continue;
}
+ let Ok(batch) = decode_prepare_slice(bytes.as_slice()) else {
+ continue;
+ };
+
+ let Some(selection) = select_batch_slice(&batch, query, matched_messages) else {
+ continue;
+ };
+ push_selected_batch_fragments(
+ &mut fragments,
+ &mut last_matching_offset,
+ &mut matched_messages,
+ &bytes,
+ &header,
+ &batch,
+ selection,
+ );
}
- messages
+ (fragments, last_matching_offset)
}
}
-impl<S> Journal<S> for PartitionJournal<S>
-where
- S: Storage<Buffer = Bytes>,
-{
+impl Journal<PartitionJournalMemStorage> for PartitionJournal<PartitionJournalMemStorage> {
type Header = PrepareHeader;
- type Entry = Message<Self::Header>;
- #[rustfmt::skip] // Scuffed formatter.
- type HeaderRef<'a> = &'a Self::Header where S: 'a;
+ type Entry = JournalBuffer;
+ #[rustfmt::skip]
+ type HeaderRef<'a> = &'a Self::Header;
fn header(&self, idx: usize) -> Option<Self::HeaderRef<'_>> {
let headers = unsafe { &mut *self.headers.get() };
@@ -355,22 +378,33 @@ where
headers.iter().find(|candidate| candidate.op == prev_op)
}
- async fn append(&self, entry: Self::Entry) -> std::io::Result<()> {
- let first_offset_and_timestamp = Self::message_to_batch(&entry)
- .and_then(|batch| Some((batch.first_offset()?, batch.first_timestamp()?)));
-
- let header = *entry.header();
+ async fn append(&self, entry: Self::Entry) -> io::Result<()> {
+ let header_size = std::mem::size_of::<PrepareHeader>();
+ let header_bytes = &entry[..header_size];
+ let header = *bytemuck::checked::try_from_bytes::<PrepareHeader>(header_bytes)
+ .expect("partition journal append expects a valid prepare header");
let op = header.op;
+ let first_offset_and_timestamp = if header.operation == Operation::SendMessages {
+ decode_prepare_slice(entry.as_slice())
+ .ok()
+ .and_then(|batch| {
+ (batch.message_count() != 0)
+ .then_some((batch.header.base_offset, batch.header.origin_timestamp))
+ })
+ } else {
+ None
+ };
{
let headers = unsafe { &mut *self.headers.get() };
headers.push(header);
};
- let bytes = entry.into_inner();
let storage_offset = {
let inner = unsafe { &*self.inner.get() };
- inner.storage.write_at(0, bytes).await?
+ let storage_offset = inner.storage.current_offset();
+ inner.storage.write_at(storage_offset, entry).await?;
+ storage_offset
};
{
@@ -383,44 +417,123 @@ where
offset_to_op.insert(offset, op);
let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() };
- timestamp_to_op.insert(timestamp, op);
+ timestamp_to_op.insert((timestamp, op), op);
}
Ok(())
}
async fn entry(&self, header: &Self::Header) -> Option<Self::Entry> {
- self.message_by_op(header.op).await
+ self.bytes_by_op(header.op).await
}
}
-impl<S> QueryableJournal<S> for PartitionJournal<S>
-where
- S: Storage<Buffer = Bytes>,
-{
+impl QueryableJournal<PartitionJournalMemStorage> for PartitionJournal<PartitionJournalMemStorage> {
type Query = MessageLookup;
- async fn get(&self, query: &Self::Query) -> Option<IggyMessagesBatchSet> {
+ async fn get(&self, query: &Self::Query) -> Option<PollQueryResult<4096>> {
let query = *query;
let start_op = self.candidate_start_op(&query)?;
- let count = match query {
- MessageLookup::Offset { count, .. } | MessageLookup::Timestamp { count, .. } => count,
- };
+ let result = self.load_polled_batches_from_storage(start_op, query).await;
+
+ if result.0.is_empty() {
+ None
+ } else {
+ Some(result)
+ }
+ }
+}
- let messages = self.load_messages_from_storage(start_op, count).await;
+fn select_batch_slice(
+ batch: &SendMessages2Ref<'_>,
+ query: MessageLookup,
+ already_matched: u32,
+) -> Option<SelectedBatchSlice> {
+ let remaining = query.count().saturating_sub(already_matched);
+ let batch_message_count = batch.message_count();
+ if remaining == 0 || batch_message_count == 0 {
+ return None;
+ }
+
+ let mut start = None;
+ let mut end = 0usize;
+ let mut matched = 0u32;
+ let mut last_matching_offset = None;
+
+ for record in batch.iter_with_offsets() {
+ let offset = batch.header.base_offset + u64::from(record.message.header.offset_delta);
- let batch_set = Self::messages_to_batch_set(&messages);
- let result = match query {
- MessageLookup::Offset { offset, count } => batch_set.get_by_offset(offset, count),
- MessageLookup::Timestamp { timestamp, count } => {
- batch_set.get_by_timestamp(timestamp, count)
+ let selected = match query {
+ MessageLookup::Offset {
+ offset: query_offset,
+ ..
+ } => offset >= query_offset,
+ MessageLookup::Timestamp { timestamp, .. } => {
+ batch.header.origin_timestamp + u64::from(record.message.header.timestamp_delta)
+ >= timestamp
}
};
+ if !selected {
+ continue;
+ }
- if result.is_empty() {
- None
- } else {
- Some(result)
+ start.get_or_insert(record.start);
+ end = record.end;
+ matched += 1;
+ last_matching_offset = Some(offset);
+
+ if matched == remaining {
+ break;
}
}
+
+ Some(SelectedBatchSlice {
+ start: start?,
+ end,
+ matched_messages: matched,
+ last_matching_offset: last_matching_offset?,
+ })
+}
+
+fn push_selected_batch_fragments(
+ fragments: &mut PollFragments<4096>,
+ last_matching_offset: &mut Option<u64>,
+ matched_messages: &mut u32,
+ prepare: &Frozen<4096>,
+ prepare_header: &PrepareHeader,
+ batch: &SendMessages2Ref<'_>,
+ selection: SelectedBatchSlice,
+) {
+ let prepare_header_size = std::mem::size_of::<PrepareHeader>();
+ let prepare_size = prepare_header.size as usize;
+ let full_body_selected = selection.start == 0 && selection.end == batch.blob().len();
+
+ if full_body_selected {
+ fragments.push(Fragment::slice(
+ prepare.clone(),
+ prepare_header_size,
+ prepare_size,
+ ));
+ } else {
+ let mut rewritten = batch.header;
+ rewritten.batch_length =
+ u64::try_from(COMMAND_HEADER_SIZE + (selection.end - selection.start))
+ .expect("sliced batch length exceeds u64::MAX");
+ rewritten.message_count = selection.matched_messages;
+ rewritten.batch_checksum = rewritten.checksum_for_blob(
+ batch
+ .blob()
+ .get(selection.start..selection.end)
+ .expect("selected batch slice must stay within blob bounds"),
+ );
+ fragments.push(Fragment::whole(rewritten.into_frozen()));
+ fragments.push(Fragment::slice(
+ prepare.clone(),
+ prepare_header_size + COMMAND_HEADER_SIZE + selection.start,
+ prepare_header_size + COMMAND_HEADER_SIZE + selection.end,
+ ));
+ }
+
+ *last_matching_offset = Some(selection.last_matching_offset);
+ *matched_messages += selection.matched_messages;
}
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs
index 67bb0b8c3..fd5bb0234 100644
--- a/core/partitions/src/lib.rs
+++ b/core/partitions/src/lib.rs
@@ -17,57 +17,26 @@
#![allow(clippy::future_not_send)]
+mod iggy_index;
+mod iggy_index_writer;
mod iggy_partition;
mod iggy_partitions;
mod journal;
mod log;
+mod messages_writer;
+mod segment;
mod types;
-use bytes::{Bytes, BytesMut};
use iggy_binary_protocol::{Message, PrepareHeader};
-use iggy_common::{
- INDEX_SIZE, IggyError, IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet, PooledBuffer,
-};
+use iggy_common::IggyError;
+pub use iggy_common::{IggyMessage2, IggyMessage2Header, IggyMessages2};
pub use iggy_partition::IggyPartition;
pub use iggy_partitions::IggyPartitions;
pub use types::{
- AppendResult, PartitionOffsets, PartitionsConfig, PollingArgs, PollingConsumer,
- SendMessagesResult,
+ AppendResult, Fragment, PartitionOffsets, PartitionsConfig, PollFragments, PollQueryResult,
+ PollingArgs, PollingConsumer, SendMessagesResult,
};
-pub(crate) fn decode_send_messages_batch(body: Bytes) -> Option<IggyMessagesBatchMut> {
- // TODO: This very is bad, IGGY-114 Fixes this.
- let mut body = body
- .try_into_mut()
- .unwrap_or_else(|body| BytesMut::from(body.as_ref()));
-
- if body.len() < 4 {
- return None;
- }
-
- let count_bytes = body.split_to(4);
- let count = u32::from_le_bytes(count_bytes.as_ref().try_into().ok()?);
- let indexes_len = (count as usize).checked_mul(INDEX_SIZE)?;
-
- if body.len() < indexes_len {
- return None;
- }
-
- let indexes_bytes = body.split_to(indexes_len);
-
- let mut indexes_buf = PooledBuffer::with_capacity(indexes_len);
- indexes_buf.put_slice(indexes_bytes.as_ref());
- let indexes = IggyIndexesMut::from_bytes(indexes_buf, 0);
-
- let messages_len = body.len();
- let mut messages = PooledBuffer::with_capacity(messages_len);
- messages.put_slice(body.as_ref());
-
- Some(IggyMessagesBatchMut::from_indexes_and_messages(
- indexes, messages,
- ))
-}
-
/// Partition-level data plane operations.
///
/// `send_messages` MUST only append to the partition journal (prepare phase),
@@ -82,7 +51,7 @@ pub trait Partition {
&self,
consumer: PollingConsumer,
args: PollingArgs,
- ) -> impl Future<Output = Result<IggyMessagesBatchSet, IggyError>> {
+ ) -> impl Future<Output = Result<PollQueryResult<4096>, IggyError>> {
let _ = (consumer, args);
async { Err(IggyError::FeatureUnavailable) }
}
diff --git a/core/partitions/src/log.rs b/core/partitions/src/log.rs
index e3798a10e..79ffd6790 100644
--- a/core/partitions/src/log.rs
+++ b/core/partitions/src/log.rs
@@ -15,13 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-use iggy_common::{
- INDEX_SIZE, IggyByteSize, IggyIndexesMut, IggyMessagesBatch, IggyMessagesBatchSetInFlight,
- Segment, SegmentStorage,
-};
+use crate::iggy_index::{IGGY_INDEX_SIZE, IggyIndexCache};
+use crate::iggy_index_writer::IggyIndexWriter;
+use crate::messages_writer::MessagesWriter;
+use crate::segment::Segment;
+use iggy_common::{IggyByteSize, IggyMessagesBatch, IggyMessagesBatchSetInFlight, SegmentStorage};
use journal::{Journal, Storage};
use ringbuffer::AllocRingBuffer;
use std::fmt::Debug;
+use std::rc::Rc;
const SEGMENTS_CAPACITY: usize = 1024;
const ACCESS_MAP_CAPACITY: usize = 8;
@@ -38,6 +40,7 @@ pub struct JournalInfo {
pub current_offset: u64,
pub first_timestamp: u64,
pub end_timestamp: u64,
+ pub max_timestamp: u64,
pub messages_count: u32,
pub size: IggyByteSize,
}
@@ -104,8 +107,10 @@ where
_access_map: AllocRingBuffer<usize>,
_cache: (),
segments: Vec<Segment>,
- indexes: Vec<Option<IggyIndexesMut>>,
+ indexes: Vec<Option<IggyIndexCache>>,
storage: Vec<SegmentStorage>,
+ messages_writers: Vec<Option<Rc<MessagesWriter>>>,
+ index_writers: Vec<Option<Rc<IggyIndexWriter>>>,
in_flight: IggyMessagesBatchSetInFlight,
}
@@ -123,6 +128,8 @@ where
segments: Vec::with_capacity(SEGMENTS_CAPACITY),
storage: Vec::with_capacity(SEGMENTS_CAPACITY),
indexes: Vec::with_capacity(SEGMENTS_CAPACITY),
+ messages_writers: Vec::with_capacity(SEGMENTS_CAPACITY),
+ index_writers: Vec::with_capacity(SEGMENTS_CAPACITY),
in_flight: IggyMessagesBatchSetInFlight::default(),
}
}
@@ -153,6 +160,22 @@ where
&self.storage
}
+ pub const fn messages_writers(&self) -> &Vec<Option<Rc<MessagesWriter>>> {
+ &self.messages_writers
+ }
+
+ pub const fn messages_writers_mut(&mut self) -> &mut Vec<Option<Rc<MessagesWriter>>> {
+ &mut self.messages_writers
+ }
+
+ pub const fn index_writers(&self) -> &Vec<Option<Rc<IggyIndexWriter>>> {
+ &self.index_writers
+ }
+
+ pub const fn index_writers_mut(&mut self) -> &mut Vec<Option<Rc<IggyIndexWriter>>> {
+ &mut self.index_writers
+ }
+
pub fn active_segment(&self) -> &Segment {
self.segments
.last()
@@ -177,22 +200,22 @@ where
.expect("active storage called on empty log")
}
- pub const fn indexes(&self) -> &Vec<Option<IggyIndexesMut>> {
+ pub const fn indexes(&self) -> &Vec<Option<IggyIndexCache>> {
&self.indexes
}
- pub const fn indexes_mut(&mut self) -> &mut Vec<Option<IggyIndexesMut>> {
+ pub const fn indexes_mut(&mut self) -> &mut Vec<Option<IggyIndexCache>> {
&mut self.indexes
}
- pub fn active_indexes(&self) -> Option<&IggyIndexesMut> {
+ pub fn active_indexes(&self) -> Option<&IggyIndexCache> {
self.indexes
.last()
.expect("active indexes called on empty log")
.as_ref()
}
- pub fn active_indexes_mut(&mut self) -> Option<&mut IggyIndexesMut> {
+ pub fn active_indexes_mut(&mut self) -> Option<&mut IggyIndexCache> {
self.indexes
.last_mut()
.expect("active indexes called on empty log")
@@ -213,18 +236,26 @@ where
.last_mut()
.expect("active indexes called on empty log");
if indexes.is_none() {
- let capacity = SIZE_16MB / INDEX_SIZE;
- *indexes = Some(IggyIndexesMut::with_capacity(capacity, 0));
+ let capacity = SIZE_16MB / IGGY_INDEX_SIZE;
+ *indexes = Some(IggyIndexCache::with_capacity(capacity));
}
}
- pub fn add_persisted_segment(&mut self, segment: Segment, storage: SegmentStorage) {
+ pub fn add_persisted_segment(
+ &mut self,
+ segment: Segment,
+ storage: SegmentStorage,
+ messages_writer: Option<Rc<MessagesWriter>>,
+ index_writer: Option<Rc<IggyIndexWriter>>,
+ ) {
self.segments.push(segment);
self.storage.push(storage);
self.indexes.push(None);
+ self.messages_writers.push(messages_writer);
+ self.index_writers.push(index_writer);
}
- pub fn set_segment_indexes(&mut self, segment_index: usize, indexes: IggyIndexesMut) {
+ pub fn set_segment_indexes(&mut self, segment_index: usize, indexes: IggyIndexCache) {
if let Some(segment_indexes) = self.indexes.get_mut(segment_index) {
*segment_indexes = Some(indexes);
}
diff --git a/core/common/src/types/segment_storage/messages_writer.rs b/core/partitions/src/messages_writer.rs
similarity index 53%
copy from core/common/src/types/segment_storage/messages_writer.rs
copy to core/partitions/src/messages_writer.rs
index 6f81f7752..330937c73 100644
--- a/core/common/src/types/segment_storage/messages_writer.rs
+++ b/core/partitions/src/messages_writer.rs
@@ -15,17 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-use super::write_batch_frozen;
-use crate::{IggyByteSize, IggyError, IggyMessagesBatch};
-use compio::fs::{File, OpenOptions};
-use err_trail::ErrContext;
+use compio::{
+ fs::{File, OpenOptions},
+ io::AsyncWriteAtExt,
+};
+use iggy_common::{IggyByteSize, IggyError};
+use iobuf::Frozen;
use std::{
rc::Rc,
sync::atomic::{AtomicU64, Ordering},
};
-use tracing::trace;
+use tracing::error;
+
+const MAX_IOV_COUNT: usize = 1024;
-/// A dedicated struct for writing to the messages file.
#[derive(Debug)]
pub struct MessagesWriter {
file_path: String,
@@ -34,15 +37,7 @@ pub struct MessagesWriter {
fsync: bool,
}
-// Safety: We are guaranteeing that MessagesWriter will never be used from multiple threads
-unsafe impl Send for MessagesWriter {}
-
impl MessagesWriter {
- /// Opens the messages file in write mode.
- ///
- /// If the server confirmation is set to `NoWait`, the file handle is transferred to the
- /// persister task (and stored in `persister_task`) so that writes are done asynchronously.
- /// Otherwise, the file is retained in `self.file` for synchronous writes.
pub async fn new(
file_path: &str,
messages_size_bytes: Rc<AtomicU64>,
@@ -54,33 +49,23 @@ impl MessagesWriter {
let file = opts
.open(file_path)
.await
- .error(|err: &std::io::Error| {
- format!("Failed to open messages file: {file_path}, error: {err}")
- })
.map_err(|_| IggyError::CannotReadFile)?;
if file_exists {
- let _ = file.sync_all().await.error(|e: &std::io::Error| {
- format!("Failed to fsync messages file after creation: {file_path}, error: {e}")
- });
+ let _ = file
+ .sync_all()
+ .await
+ .map_err(|_| IggyError::CannotWriteToFile);
let actual_messages_size = file
.metadata()
.await
- .error(|e: &std::io::Error| {
- format!("Failed to get metadata of messages file: {file_path}, error: {e}")
- })
.map_err(|_| IggyError::CannotReadFileMetadata)?
.len();
messages_size_bytes.store(actual_messages_size, Ordering::Relaxed);
}
- trace!(
- "Opened messages file for writing: {file_path}, size: {}",
- messages_size_bytes.load(Ordering::Acquire)
- );
-
Ok(Self {
file_path: file_path.to_string(),
file,
@@ -89,27 +74,21 @@ impl MessagesWriter {
})
}
- /// Append frozen (immutable) batches to the messages file.
- /// The caller retains the batches (for use in in-flight buffer) while disk I/O proceeds.
- pub async fn save_frozen_batches(
+ pub async fn save_frozen_batches<const ALIGN: usize>(
&self,
- batches: &[IggyMessagesBatch],
+ buffers: &[Frozen<ALIGN>],
) -> Result<IggyByteSize, IggyError> {
- let messages_size: u64 = batches.iter().map(|b| b.size() as u64).sum();
+ let messages_size: u64 = buffers.iter().map(|buffer| buffer.len() as u64).sum();
+
+ if messages_size == 0 {
+ return Ok(IggyByteSize::from(0));
+ }
let position = self.messages_size_bytes.load(Ordering::Relaxed);
- let file = &self.file;
- write_batch_frozen(file, position, batches)
- .await
- .error(|e: &IggyError| {
- format!(
- "Failed to write frozen batch to messages file: {}. {e}",
- self.file_path
- )
- })?;
+ write_frozen_chunked(&self.file, &self.file_path, position, buffers).await?;
if self.fsync {
- let _ = self.fsync().await;
+ self.fsync().await?;
}
self.messages_size_bytes
@@ -126,10 +105,38 @@ impl MessagesWriter {
self.file
.sync_all()
.await
- .error(|e: &std::io::Error| {
- format!("Failed to fsync messages file: {}. {e}", self.file_path)
- })
.map_err(|_| IggyError::CannotWriteToFile)?;
Ok(())
}
}
+
+async fn write_frozen_chunked<const ALIGN: usize>(
+ file: &File,
+ file_path: &str,
+ mut position: u64,
+ buffers: &[Frozen<ALIGN>],
+) -> Result<(), IggyError> {
+ for chunk in buffers.chunks(MAX_IOV_COUNT) {
+ let chunk_size: usize = chunk.iter().map(Frozen::len).sum();
+ let chunk_vec: Vec<_> = chunk.to_vec();
+
+ let (result, _) = (&*file)
+ .write_vectored_all_at(chunk_vec, position)
+ .await
+ .into();
+ result.map_err(|err| {
+ error!(
+ target: "iggy.partitions.storage",
+ file = file_path,
+ write_position = position,
+ %err,
+ "failed to write frozen messages to segment file"
+ );
+ IggyError::CannotWriteToFile
+ })?;
+
+ position += chunk_size as u64;
+ }
+
+ Ok(())
+}
diff --git a/core/partitions/src/segment.rs b/core/partitions/src/segment.rs
new file mode 100644
index 000000000..69293f286
--- /dev/null
+++ b/core/partitions/src/segment.rs
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use iggy_common::{IggyByteSize, IggyExpiry, IggyTimestamp};
+use std::fmt::Display;
+
+#[derive(Default, Debug, Clone)]
+pub struct Segment {
+ pub sealed: bool,
+ pub start_timestamp: u64,
+ pub end_timestamp: u64,
+ pub max_timestamp: u64,
+ pub current_position: u64,
+ pub start_offset: u64,
+ pub end_offset: u64,
+ pub size: IggyByteSize,
+ pub max_size: IggyByteSize,
+}
+
+impl Display for Segment {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "Segment {{ sealed: {}, max_size: {}, start_timestamp: {}, end_timestamp: {}, max_timestamp: {}, start_offset: {}, end_offset: {}, size: {} }}",
+ self.sealed,
+ self.max_size,
+ self.start_timestamp,
+ self.end_timestamp,
+ self.max_timestamp,
+ self.start_offset,
+ self.end_offset,
+ self.size
+ )
+ }
+}
+
+impl Segment {
+ #[must_use]
+ pub fn new(start_offset: u64, max_size_bytes: IggyByteSize) -> Self {
+ Self {
+ sealed: false,
+ max_size: max_size_bytes,
+ start_timestamp: 0,
+ end_timestamp: 0,
+ max_timestamp: 0,
+ start_offset,
+ end_offset: start_offset,
+ size: IggyByteSize::default(),
+ current_position: 0,
+ }
+ }
+
+ #[must_use]
+ pub fn is_full(&self) -> bool {
+ self.current_position >= self.max_size.as_bytes_u64() || self.size >= self.max_size
+ }
+
+ #[must_use]
+ pub fn is_expired(&self, now: IggyTimestamp, expiry: IggyExpiry) -> bool {
+ if !self.sealed {
+ return false;
+ }
+
+ match expiry {
+ IggyExpiry::NeverExpire | IggyExpiry::ServerDefault => false,
+ IggyExpiry::ExpireDuration(duration) => {
+ self.max_timestamp + duration.as_micros() <= now.as_micros()
+ }
+ }
+ }
+}
diff --git a/core/common/src/types/mod.rs b/core/partitions/src/send_messages2.rs
similarity index 57%
copy from core/common/src/types/mod.rs
copy to core/partitions/src/send_messages2.rs
index 01b19de7b..5cd17fb5a 100644
--- a/core/common/src/types/mod.rs
+++ b/core/partitions/src/send_messages2.rs
@@ -15,26 +15,3 @@
// specific language governing permissions and limitations
// under the License.
-pub(crate) mod args;
-pub(crate) mod client;
-pub(crate) mod client_state;
-pub(crate) mod cluster;
-pub(crate) mod command;
-pub(crate) mod compression;
-pub(crate) mod configuration;
-pub(crate) mod consumer;
-pub(crate) mod diagnostic;
-pub(crate) mod either;
-pub(crate) mod identifier;
-pub(crate) mod message;
-pub(crate) mod partition;
-pub(crate) mod permissions;
-pub(crate) mod personal_access_tokens;
-pub(crate) mod segment;
-pub(crate) mod segment_storage;
-pub(crate) mod snapshot;
-pub(crate) mod stats;
-pub(crate) mod stream;
-pub(crate) mod streaming_stats;
-pub(crate) mod topic;
-pub(crate) mod user;
diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs
index 1a21609a5..dbaf57a59 100644
--- a/core/partitions/src/types.rs
+++ b/core/partitions/src/types.rs
@@ -16,6 +16,46 @@
// under the License.
use iggy_common::{IggyByteSize, PollingStrategy};
+use iobuf::Frozen;
+use smallvec::SmallVec;
+
+#[derive(Debug, Clone)]
+pub struct Fragment<const ALIGN: usize = 4096> {
+ source: Frozen<ALIGN>,
+ start: usize,
+ end: usize,
+}
+
+impl<const ALIGN: usize> Fragment<ALIGN> {
+ #[must_use]
+ pub fn whole(source: Frozen<ALIGN>) -> Self {
+ let end = source.len();
+ Self {
+ source,
+ start: 0,
+ end,
+ }
+ }
+
+ #[must_use]
+ /// # Panics
+ ///
+ /// Panics if `start > end` or if `end` is past the end of `source`.
+ pub fn slice(source: Frozen<ALIGN>, start: usize, end: usize) -> Self {
+ assert!(start <= end);
+ assert!(end <= source.len());
+ Self { source, start, end }
+ }
+
+ #[must_use]
+ pub fn into_frozen(self) -> Frozen<ALIGN> {
+ if self.start == 0 && self.end == self.source.len() {
+ self.source
+ } else {
+ self.source.slice(self.start..self.end)
+ }
+ }
+}
/// Arguments for polling messages from a partition.
#[derive(Debug, Clone)]
@@ -25,6 +65,9 @@ pub struct PollingArgs {
pub auto_commit: bool,
}
+pub type PollFragments<const ALIGN: usize = 4096> = SmallVec<[Fragment<ALIGN>; 4]>;
+pub type PollQueryResult<const ALIGN: usize = 4096> = (PollFragments<ALIGN>, Option<u64>);
+
impl PollingArgs {
#[must_use]
pub const fn new(strategy: PollingStrategy, count: u32, auto_commit: bool) -> Self {
@@ -85,32 +128,28 @@ impl AppendResult {
/// Current offset state of a partition.
///
-/// Tracks both the commit offset (visibility boundary) and write offset
-/// (highest written message). These may differ when there are prepared
-/// but uncommitted messages.
+/// Tracks both the durable offset (highest persisted message) and write offset
+/// (highest assigned message offset). These may differ when there are prepared
+/// messages that still only live in the in-memory journal.
///
/// ```text
/// Segment: [msg0][msg1][msg2][msg3][msg4][msg5][msg6][msg7]
/// ▲ ▲
-/// commit_offset write_offset
+/// durable_offset write_offset
/// (4) (7)
///
-/// - Messages 0-4: COMMITTED (visible to consumers)
-/// - Messages 5-7: PREPARED but not committed (invisible)
+/// - Messages 0-4: durably persisted
+/// - Messages 5-7: prepared, but still buffered in memory
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PartitionOffsets {
- /// Highest offset visible to consumers.
- ///
- /// All messages with `offset <= commit_offset` can be read via
- /// `read_committed()` or `poll_messages()`.
+ /// Highest durably persisted offset.
pub commit_offset: u64,
- /// Highest offset written to storage.
+ /// Highest offset assigned to the partition.
///
/// This may be greater than `commit_offset` when there are prepared
- /// but uncommitted messages (during the window between prepare and
- /// commit in VSR).
+ /// messages buffered in the in-memory journal.
///
/// Invariant: `write_offset >= commit_offset`
pub write_offset: u64,
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index c6863a25d..a38a79143 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -76,6 +76,7 @@ impl<R: Send + 'static> ShardFrame<R> {
/// Create a request-response frame. Returns the frame and a receiver
/// that the caller can await for completion notification.
+ #[must_use]
pub fn with_response(message: Message<GenericHeader>) -> (Self, Receiver<R>) {
let (tx, rx) = channel(1);
(
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index 7398e9e2d..8965a6a05 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -51,17 +51,17 @@ where
}
};
let (operation, namespace, generic) = match bag {
- MessageBag::Request(ref r) => {
- let h = r.header();
- (h.operation, h.namespace, r.as_generic().clone())
+ MessageBag::Request(r) => {
+ let h = *r.header();
+ (h.operation, h.namespace, r.into_generic())
}
- MessageBag::Prepare(ref p) => {
- let h = p.header();
- (h.operation, h.namespace, p.as_generic().clone())
+ MessageBag::Prepare(p) => {
+ let h = *p.header();
+ (h.operation, h.namespace, p.into_generic())
}
- MessageBag::PrepareOk(ref p) => {
- let h = p.header();
- (h.operation, h.namespace, p.as_generic().clone())
+ MessageBag::PrepareOk(p) => {
+ let h = *p.header();
+ (h.operation, h.namespace, p.into_generic())
}
};
let namespace = IggyNamespace::from_raw(namespace);
@@ -95,17 +95,17 @@ where
) -> Result<Receiver<R>, ConsensusError> {
let bag = MessageBag::try_from(message)?;
let (operation, namespace, generic) = match bag {
- MessageBag::Request(ref r) => {
- let h = r.header();
- (h.operation, h.namespace, r.as_generic().clone())
+ MessageBag::Request(r) => {
+ let h = *r.header();
+ (h.operation, h.namespace, r.into_generic())
}
- MessageBag::Prepare(ref p) => {
- let h = p.header();
- (h.operation, h.namespace, p.as_generic().clone())
+ MessageBag::Prepare(p) => {
+ let h = *p.header();
+ (h.operation, h.namespace, p.into_generic())
}
- MessageBag::PrepareOk(ref p) => {
- let h = p.header();
- (h.operation, h.namespace, p.as_generic().clone())
+ MessageBag::PrepareOk(p) => {
+ let h = *p.header();
+ (h.operation, h.namespace, p.into_generic())
}
};
let namespace = IggyNamespace::from_raw(namespace);
diff --git a/core/simulator/Cargo.toml b/core/simulator/Cargo.toml
index 556e47388..7c3c93c74 100644
--- a/core/simulator/Cargo.toml
+++ b/core/simulator/Cargo.toml
@@ -28,6 +28,7 @@ enumset = { workspace = true }
futures = { workspace = true }
iggy_binary_protocol = { path = "../binary_protocol" }
iggy_common = { path = "../common" }
+iobuf = { workspace = true }
journal = { path = "../journal" }
message_bus = { path = "../message_bus" }
metadata = { path = "../metadata" }
diff --git a/core/simulator/src/bus.rs b/core/simulator/src/bus.rs
index 93c9ef0f6..7781f5dbf 100644
--- a/core/simulator/src/bus.rs
+++ b/core/simulator/src/bus.rs
@@ -23,12 +23,18 @@ use std::ops::Deref;
use std::sync::{Arc, Mutex};
/// Message envelope for tracking sender/recipient
-#[derive(Debug, Clone)]
+#[derive(Debug)]
+pub enum EnvelopePayload {
+ Replica(Message<GenericHeader>),
+ Client(Message<GenericHeader>),
+}
+
+#[derive(Debug)]
pub struct Envelope {
pub from_replica: Option<u8>,
pub to_replica: Option<u8>,
pub to_client: Option<u128>,
- pub message: Message<GenericHeader>,
+ pub payload: EnvelopePayload,
}
// TODO: Proper bus with an `Network` component which would simulate sending packets.
@@ -95,7 +101,7 @@ impl MessageBus for MemBus {
&self,
client_id: Self::Client,
message: Self::Data,
- ) -> Result<(), IggyError> {
+ ) -> Result<Self::Data, IggyError> {
if !self.clients.lock().unwrap().contains(&client_id) {
#[allow(clippy::cast_possible_truncation)]
return Err(IggyError::ClientNotFound(client_id as u32));
@@ -105,17 +111,17 @@ impl MessageBus for MemBus {
from_replica: None,
to_replica: None,
to_client: Some(client_id),
- message,
+ payload: EnvelopePayload::Client(message.deep_copy()),
});
- Ok(())
+ Ok(message)
}
async fn send_to_replica(
&self,
replica: Self::Replica,
message: Self::Data,
- ) -> Result<(), IggyError> {
+ ) -> Result<Self::Data, IggyError> {
if !self.replicas.lock().unwrap().contains(&replica) {
return Err(IggyError::ResourceNotFound(format!("Replica {replica}")));
}
@@ -124,10 +130,10 @@ impl MessageBus for MemBus {
from_replica: None,
to_replica: Some(replica),
to_client: None,
- message,
+ payload: EnvelopePayload::Replica(message.deep_copy()),
});
- Ok(())
+ Ok(message)
}
}
@@ -168,7 +174,7 @@ impl MessageBus for SharedMemBus {
&self,
client_id: Self::Client,
message: Self::Data,
- ) -> Result<(), IggyError> {
+ ) -> Result<Self::Data, IggyError> {
self.0.send_to_client(client_id, message).await
}
@@ -176,7 +182,7 @@ impl MessageBus for SharedMemBus {
&self,
replica: Self::Replica,
message: Self::Data,
- ) -> Result<(), IggyError> {
+ ) -> Result<Self::Data, IggyError> {
self.0.send_to_replica(replica, message).await
}
}
diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs
index 2a73de834..2d64d0f40 100644
--- a/core/simulator/src/client.rs
+++ b/core/simulator/src/client.rs
@@ -15,11 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+use bytes::Bytes;
use iggy_binary_protocol::{Message, Operation, RequestHeader};
+use iggy_common::send_messages2::{
+ IggyMessage2, IggyMessage2Header, IggyMessages2, SendMessages2Owned,
+};
use iggy_common::{
- BytesSerializable, IGGY_MESSAGE_HEADER_SIZE, INDEX_SIZE, Identifier,
- create_stream::CreateStream, delete_stream::DeleteStream, sharding::IggyNamespace,
+ BytesSerializable, Identifier, create_stream::CreateStream, delete_stream::DeleteStream,
+ sharding::IggyNamespace,
};
+use iobuf::Owned;
use std::cell::Cell;
// TODO: Proper client which implements the full client SDK API
@@ -64,45 +69,34 @@ impl SimClient {
}
#[allow(clippy::cast_possible_truncation)]
+ /// # Panics
+ ///
+ /// Panics if the simulator cannot encode the provided messages into a valid
+ /// `SendMessages2` request.
pub fn send_messages(
&self,
namespace: IggyNamespace,
messages: &[&[u8]],
) -> Message<RequestHeader> {
- let count = messages.len() as u32;
- let mut indexes = Vec::with_capacity(count as usize * INDEX_SIZE);
- let mut messages_buf = Vec::new();
-
- let mut current_position = 0u32;
- for (i, msg) in messages.iter().enumerate() {
- let msg_total_len = (IGGY_MESSAGE_HEADER_SIZE + msg.len()) as u32;
-
- // Index: offset(u32) + position(u32) + timestamp(u64)
- indexes.extend_from_slice(&(i as u32).to_le_bytes()); // offset (relative)
- indexes.extend_from_slice(¤t_position.to_le_bytes()); // position
- indexes.extend_from_slice(&0u64.to_le_bytes()); // timestamp (set in prepare)
-
- // Message header (64 bytes)
- messages_buf.extend_from_slice(&0u64.to_le_bytes()); // checksum
- messages_buf.extend_from_slice(&0u128.to_le_bytes()); // id
- messages_buf.extend_from_slice(&0u64.to_le_bytes()); // offset
- messages_buf.extend_from_slice(&0u64.to_le_bytes()); // timestamp
- messages_buf.extend_from_slice(&0u64.to_le_bytes()); // origin_timestamp
- messages_buf.extend_from_slice(&0u32.to_le_bytes()); // user_headers_length
- messages_buf.extend_from_slice(&(msg.len() as u32).to_le_bytes()); // payload_length
- messages_buf.extend_from_slice(&0u64.to_le_bytes()); // reserved
-
- // Payload
- messages_buf.extend_from_slice(msg);
- current_position += msg_total_len;
+ let mut batch = IggyMessages2::with_capacity(messages.len());
+ for message in messages {
+ batch.push(IggyMessage2 {
+ header: IggyMessage2Header {
+ payload_length: message.len() as u32,
+ ..Default::default()
+ },
+ payload: Bytes::copy_from_slice(message),
+ user_headers: None,
+ });
}
- let mut payload = Vec::with_capacity(4 + indexes.len() + messages_buf.len());
- payload.extend_from_slice(&count.to_le_bytes());
- payload.extend_from_slice(&indexes);
- payload.extend_from_slice(&messages_buf);
-
- self.build_request_with_namespace(Operation::SendMessages, &payload, namespace)
+ let batch = SendMessages2Owned::from_messages(namespace, &batch)
+ .expect("simulator must build a valid send_messages2 batch");
+ let total_size = std::mem::size_of::<RequestHeader>() + batch.header.total_size();
+ let request_header = self.request_header(Operation::SendMessages, namespace, total_size);
+ batch
+ .encode_request(request_header)
+ .expect("simulator must build a valid send_messages2 request")
}
pub fn store_consumer_offset(
@@ -127,8 +121,22 @@ impl SimClient {
payload: &[u8],
namespace: IggyNamespace,
) -> Message<RequestHeader> {
- use bytes::Bytes;
+ let header_size = std::mem::size_of::<RequestHeader>();
+ let total_size = header_size + payload.len();
+
+ let header = self.request_header(operation, namespace, total_size);
+
+ let header_bytes = bytemuck::bytes_of(&header);
+ let mut buffer = Vec::with_capacity(total_size);
+ buffer.extend_from_slice(header_bytes);
+ buffer.extend_from_slice(payload);
+ Message::try_from(Owned::<4096>::copy_from_slice(&buffer))
+ .expect("request buffer must contain a valid request message")
+ }
+
+ #[allow(clippy::cast_possible_truncation)]
+ fn build_request(&self, operation: Operation, payload: &[u8]) -> Message<RequestHeader> {
let header_size = std::mem::size_of::<RequestHeader>();
let total_size = header_size + payload.len();
@@ -136,7 +144,7 @@ impl SimClient {
command: iggy_binary_protocol::Command2::Request,
operation,
size: total_size as u32,
- cluster: 0,
+ cluster: 0, // TODO: Get from config
checksum: 0,
checksum_body: 0,
view: 0,
@@ -145,9 +153,8 @@ impl SimClient {
reserved_frame: [0; 66],
client: self.client_id,
request_checksum: 0,
- timestamp: 0,
+ timestamp: 0, // TODO: Use actual timestamp
request: self.next_request_number(),
- namespace: namespace.inner(),
..Default::default()
};
@@ -156,22 +163,22 @@ impl SimClient {
buffer.extend_from_slice(header_bytes);
buffer.extend_from_slice(payload);
- Message::<RequestHeader>::from_bytes(Bytes::from(buffer))
- .expect("failed to build request message")
+ Message::try_from(Owned::<4096>::copy_from_slice(&buffer))
+ .expect("request buffer must contain a valid request message")
}
#[allow(clippy::cast_possible_truncation)]
- fn build_request(&self, operation: Operation, payload: &[u8]) -> Message<RequestHeader> {
- use bytes::Bytes;
-
- let header_size = std::mem::size_of::<RequestHeader>();
- let total_size = header_size + payload.len();
-
- let header = RequestHeader {
+ fn request_header(
+ &self,
+ operation: Operation,
+ namespace: IggyNamespace,
+ total_size: usize,
+ ) -> RequestHeader {
+ RequestHeader {
command: iggy_binary_protocol::Command2::Request,
operation,
size: total_size as u32,
- cluster: 0, // TODO: Get from config
+ cluster: 0,
checksum: 0,
checksum_body: 0,
view: 0,
@@ -180,17 +187,10 @@ impl SimClient {
reserved_frame: [0; 66],
client: self.client_id,
request_checksum: 0,
- timestamp: 0, // TODO: Use actual timestamp
+ timestamp: 0,
request: self.next_request_number(),
+ namespace: namespace.inner(),
..Default::default()
- };
-
- let header_bytes = bytemuck::bytes_of(&header);
- let mut buffer = Vec::with_capacity(total_size);
- buffer.extend_from_slice(header_bytes);
- buffer.extend_from_slice(payload);
-
- Message::<RequestHeader>::from_bytes(Bytes::from(buffer))
- .expect("failed to build request message")
+ }
}
}
diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs
index e6ca06b89..d3410e311 100644
--- a/core/simulator/src/deps.rs
+++ b/core/simulator/src/deps.rs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use bytes::Bytes;
use iggy_binary_protocol::{Message, PrepareHeader};
use iggy_common::variadic;
+use iobuf::Owned;
use journal::{Journal, JournalHandle, Storage};
use metadata::MuxStateMachine;
use metadata::stm::consumer_group::ConsumerGroups;
@@ -112,10 +112,13 @@ impl<S: Storage<Buffer = Vec<u8>>> Journal<S> for SimJournal<S> {
let header = headers.get(&header.op)?;
let offset = *offsets.get(&header.op)?;
- let buffer = vec![0; header.size as usize];
- let buffer = self.storage.read_at(offset, buffer).await.ok()?;
- let message =
- Message::from_bytes(Bytes::from(buffer)).expect("simulator: bytes should be valid");
+ let buffer = self
+ .storage
+ .read_at(offset, vec![0; header.size as usize])
+ .await
+ .ok()?;
+ let message = Message::try_from(Owned::<4096>::copy_from_slice(&buffer))
+ .expect("prepare buffer must contain a valid prepare message");
Some(message)
}
@@ -128,14 +131,13 @@ impl<S: Storage<Buffer = Vec<u8>>> Journal<S> for SimJournal<S> {
async fn append(&self, entry: Self::Entry) -> std::io::Result<()> {
let header = *entry.header();
- let message_bytes = entry.as_bytes();
+ let message_bytes = entry.into_frozen();
+ let offset = self.write_offset.get();
let bytes_written = self
.storage
- .write_at(self.write_offset.get(), message_bytes.to_vec())
+ .write_at(offset, message_bytes.as_slice().to_vec())
.await?;
-
- let offset = self.write_offset.get();
unsafe { &mut *self.headers.get() }.insert(header.op, header);
unsafe { &mut *self.offsets.get() }.insert(header.op, offset);
self.write_offset.set(offset + bytes_written);
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 21effc797..95cdb36ec 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -23,13 +23,13 @@ pub mod packet;
pub mod ready_queue;
pub mod replica;
-use bus::MemBus;
+use bus::{EnvelopePayload, MemBus};
use consensus::PartitionsHandle;
use iggy_binary_protocol::{GenericHeader, Message, ReplyHeader};
+use iggy_common::IggyError;
use iggy_common::sharding::IggyNamespace;
-use iggy_common::{IggyError, IggyMessagesBatchSet};
use message_bus::MessageBus;
-use partitions::{Partition, PartitionOffsets, PollingArgs, PollingConsumer};
+use partitions::{Partition, PartitionOffsets, PollQueryResult, PollingArgs, PollingConsumer};
use replica::{Replica, new_replica};
use std::sync::Arc;
@@ -107,17 +107,22 @@ impl Simulator {
pub async fn step(&self) -> Option<Message<ReplyHeader>> {
if let Some(envelope) = self.message_bus.receive() {
if let Some(_client_id) = envelope.to_client {
- let reply: Message<ReplyHeader> = envelope
- .message
+ let EnvelopePayload::Client(message) = envelope.payload else {
+ panic!("client envelope must carry a reply message");
+ };
+ let reply: Message<ReplyHeader> = message
.try_into_typed()
- .expect("invalid message, wrong command type for an client response");
+ .expect("invalid message, wrong command type for a client response");
return Some(reply);
}
if let Some(replica_id) = envelope.to_replica
&& let Some(replica) = self.replicas.get(replica_id as usize)
{
- self.dispatch_to_replica(replica, envelope.message).await;
+ let EnvelopePayload::Replica(message) = envelope.payload else {
+ panic!("replica envelope must carry a replica message");
+ };
+ self.dispatch_to_replica(replica, message).await;
}
}
@@ -150,7 +155,7 @@ impl Simulator {
namespace: IggyNamespace,
consumer: PollingConsumer,
args: PollingArgs,
- ) -> Result<IggyMessagesBatchSet, IggyError> {
+ ) -> Result<PollQueryResult<4096>, IggyError> {
let replica = &self.replicas[replica_idx];
let partition =
replica
diff --git a/core/simulator/src/main.rs b/core/simulator/src/main.rs
index cffd0a38f..fe91cda8e 100644
--- a/core/simulator/src/main.rs
+++ b/core/simulator/src/main.rs
@@ -145,11 +145,8 @@ fn main() {
.poll_messages(leader as usize, test_namespace, consumer, args)
.await
{
- Ok(batch_set) => {
- println!(
- "[sim] Poll returned {} messages (expected 3)",
- batch_set.count()
- );
+ Ok((fragments, _last_matching_offset)) => {
+ println!("[sim] Poll returned {} fragments", fragments.len());
}
Err(e) => {
println!("[sim] Poll failed: {e}");
@@ -161,7 +158,10 @@ fn main() {
.poll_messages(leader as usize, test_namespace, consumer, args_auto)
.await
{
- println!("[sim] Auto-commit poll returned {} messages", batch.count());
+ println!(
+ "[sim] Auto-commit poll returned {} fragments",
+ batch.0.len()
+ );
}
// Next poll should start from offset 2 (after auto-commit of 0,1)
@@ -170,10 +170,7 @@ fn main() {
.poll_messages(leader as usize, test_namespace, consumer, args_next)
.await
{
- println!(
- "[sim] Next poll returned {} messages (expected 1)",
- batch.count()
- );
+ println!("[sim] Next poll returned {} fragments", batch.0.len());
}
// Check offsets
diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs
index 1658628de..171e0d20a 100644
--- a/core/simulator/src/packet.rs
+++ b/core/simulator/src/packet.rs
@@ -66,7 +66,7 @@ pub enum ProcessId {
}
/// A packet in flight through the simulated network.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct Packet {
pub from: ProcessId,
pub to: ProcessId,
@@ -75,6 +75,17 @@ pub struct Packet {
pub ready_at: u64,
}
+impl Clone for Packet {
+ fn clone(&self) -> Self {
+ Self {
+ from: self.from,
+ to: self.to,
+ message: self.message.deep_copy(),
+ ready_at: self.ready_at,
+ }
+ }
+}
+
impl Ready for Packet {
fn ready_at(&self) -> u64 {
self.ready_at
@@ -743,7 +754,8 @@ mod tests {
let header: &mut GenericHeader =
bytemuck::checked::try_from_bytes_mut(&mut buf).expect("zeroed bytes are valid");
header.command = command;
- Message::<GenericHeader>::from_bytes(bytes::Bytes::from(buf)).unwrap()
+ Message::try_from(iobuf::Owned::<4096>::copy_from_slice(&buf))
+ .expect("generic test buffer must contain a valid generic message")
}
/// Helper: disable all links to/from a given replica (isolate it).
@@ -807,7 +819,11 @@ mod tests {
let msg = create_test_message();
// Send packet during partition
- sim.submit(ProcessId::Replica(0), ProcessId::Replica(1), msg.clone());
+ sim.submit(
+ ProcessId::Replica(0),
+ ProcessId::Replica(1),
+ msg.deep_copy(),
+ );
for _ in 0..20 {
sim.tick();
sim.step(); // drain and drop