You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/11/18 16:41:34 UTC
(iggy) branch master updated: feat(cluster): prepare consensus header setup and add clock trait (#2371)
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 1151b95a6 feat(cluster): prepare consensus header setup and add clock trait (#2371)
1151b95a6 is described below
commit 1151b95a6ca808b8e9cbd7949a777cf7215b727d
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Tue Nov 18 17:41:29 2025 +0100
feat(cluster): prepare consensus header setup and add clock trait (#2371)
---
Cargo.lock | 12 ++++
Cargo.toml | 1 +
DEPENDENCIES.md | 1 +
core/{consensus => clock}/Cargo.toml | 11 +---
core/{consensus/src/lib.rs => clock/src/impls.rs} | 15 +++--
core/{consensus => clock}/src/lib.rs | 10 ++--
.../src/types/consensus/header.rs} | 9 ++-
core/common/src/types/consensus/mod.rs | 70 ++++++++++++++++++----
core/consensus/Cargo.toml | 3 +
core/consensus/src/{lib.rs => impls.rs} | 6 +-
core/consensus/src/lib.rs | 22 ++++++-
core/metadata/Cargo.toml | 2 +
core/metadata/src/impls/metadata.rs | 32 +++++++---
13 files changed, 146 insertions(+), 48 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 5f7850cbc..3030058f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1413,6 +1413,13 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
+[[package]]
+name = "clock"
+version = "0.1.0"
+dependencies = [
+ "iggy_common",
+]
+
[[package]]
name = "cmake"
version = "0.1.54"
@@ -1696,6 +1703,9 @@ dependencies = [
[[package]]
name = "consensus"
version = "0.1.0"
+dependencies = [
+ "clock",
+]
[[package]]
name = "console"
@@ -5221,7 +5231,9 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
name = "metadata"
version = "0.1.0"
dependencies = [
+ "clock",
"consensus",
+ "tracing",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 53e3ae956..811ec468d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,6 +27,7 @@ members = [
"core/bench/runner",
"core/binary_protocol",
"core/cli",
+ "core/clock",
"core/common",
"core/connectors/runtime",
"core/connectors/sdk",
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 69c20d188..061e2ca25 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -122,6 +122,7 @@ clap_builder: 4.5.51, "Apache-2.0 OR MIT",
clap_complete: 4.5.60, "Apache-2.0 OR MIT",
clap_derive: 4.5.49, "Apache-2.0 OR MIT",
clap_lex: 0.7.6, "Apache-2.0 OR MIT",
+clock: 0.1.0, "N/A",
cmake: 0.1.54, "Apache-2.0 OR MIT",
cobs: 0.3.0, "Apache-2.0 OR MIT",
colorchoice: 1.0.4, "Apache-2.0 OR MIT",
diff --git a/core/consensus/Cargo.toml b/core/clock/Cargo.toml
similarity index 70%
copy from core/consensus/Cargo.toml
copy to core/clock/Cargo.toml
index 9bf2b9ad8..674f5bf8a 100644
--- a/core/consensus/Cargo.toml
+++ b/core/clock/Cargo.toml
@@ -16,16 +16,9 @@
# under the License.
[package]
-name = "consensus"
+name = "clock"
version = "0.1.0"
-description = "Iggy consensus module"
edition = "2024"
-license = "Apache-2.0"
-keywords = ["iggy", "messaging", "streaming"]
-categories = ["command-line-utilities", "database", "network-programming"]
-homepage = "https://iggy.apache.org"
-documentation = "https://iggy.apache.org/docs"
-repository = "https://github.com/apache/iggy"
-readme = "../../../README.md"
[dependencies]
+iggy_common = { workspace = true }
diff --git a/core/consensus/src/lib.rs b/core/clock/src/impls.rs
similarity index 77%
copy from core/consensus/src/lib.rs
copy to core/clock/src/impls.rs
index c3241d3c7..6b34f289e 100644
--- a/core/consensus/src/lib.rs
+++ b/core/clock/src/impls.rs
@@ -15,8 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-pub trait Consensus {
- type RequestMessage;
- type ReplicateMessage;
- type AckMessage;
+use crate::Clock;
+use iggy_common::IggyTimestamp;
+
+pub struct IggySystemClock;
+
+impl Clock for IggySystemClock {
+ type Realtime = IggyTimestamp;
+
+ fn realtime(&self) -> Self::Realtime {
+ IggyTimestamp::now()
+ }
}
diff --git a/core/consensus/src/lib.rs b/core/clock/src/lib.rs
similarity index 86%
copy from core/consensus/src/lib.rs
copy to core/clock/src/lib.rs
index c3241d3c7..2223894ed 100644
--- a/core/consensus/src/lib.rs
+++ b/core/clock/src/lib.rs
@@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-pub trait Consensus {
- type RequestMessage;
- type ReplicateMessage;
- type AckMessage;
+pub trait Clock {
+ type Realtime;
+ fn realtime(&self) -> Self::Realtime;
}
+
+mod impls;
+pub use impls::IggySystemClock;
diff --git a/core/consensus/src/lib.rs b/core/common/src/types/consensus/header.rs
similarity index 89%
copy from core/consensus/src/lib.rs
copy to core/common/src/types/consensus/header.rs
index c3241d3c7..96ad1eaa0 100644
--- a/core/consensus/src/lib.rs
+++ b/core/common/src/types/consensus/header.rs
@@ -15,8 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-pub trait Consensus {
- type RequestMessage;
- type ReplicateMessage;
- type AckMessage;
-}
+#[expect(unused)]
+pub struct Header {}
+
+pub trait ConsensusHeader {}
diff --git a/core/common/src/types/consensus/mod.rs b/core/common/src/types/consensus/mod.rs
index 6ecac7726..795695d12 100644
--- a/core/common/src/types/consensus/mod.rs
+++ b/core/common/src/types/consensus/mod.rs
@@ -17,6 +17,10 @@
use std::marker::PhantomData;
+use bytes::Bytes;
+
+use crate::types::consensus::header::ConsensusHeader;
+
// TODO: Header generic
// TODO: We will have to impl something like this (NOT ONE TO ONE JUST A SKETCH) as we will use `bytemuck`:
/*
@@ -40,14 +44,20 @@ pub trait Header: Sized {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum Command {
- Reserved = 0,
- Ping = 1,
- Pong = 2,
- Request = 3,
- Prepare = 4,
- PrepareOk = 5,
- Reply = 6,
- Commit = 7,
+ reserved = 0,
+
+ ping = 1,
+ pong = 2,
+
+ request = 5,
+ prepare = 6,
+ prepare_ok = 7,
+ reply = 8,
+ commit = 9,
+
+ start_view_change = 10,
+ do_view_change = 11,
+ start_view = 24,
// ... etc
}
@@ -60,6 +70,9 @@ pub struct GenericHeader {
// ... other fields
}
+// first 128 bytes GenericHeader (checksum, command, size, etc....)
+// second 128 bytes specific header (PrepareHeader, CommitHeader, etc....)
+
impl Header for GenericHeader {
const COMMAND: Command = Command::Reserved;
@@ -86,6 +99,7 @@ impl Header for PrepareHeader {
fn size(&self) -> u32 { self.size }
fn command(&self) -> Command { self.command }
fn checksum(&self) -> u128 { self.checksum }
+}
*/
// And then for Message impl
@@ -152,11 +166,47 @@ pub enum Operation {
Consume = 101,
Fetch = 102,
}
+
+// Specific header types
+#[repr(C)]
+pub struct PrepareHeader {
+ checksum: u128,
+ command: Command,
+ size: u32,
+ // ... prepare-specific fields
+ view: u32,
+ op: u64,
+ commit: u64,
+
+ // second 128 bytes
+ operation: Operation,
+}
*/
// Which will have an method that returns discriminator between Metadata and Partition requests
+// TODO: Fill this enum
#[expect(unused)]
-pub struct Message<H> {
- _phantom_header: PhantomData<H>,
+pub enum MessageBag {
+ Void,
}
+
+#[expect(unused)]
+pub struct Message<H>
+where
+ H: ConsensusHeader,
+{
+ buffer: Bytes,
+ _h: PhantomData<H>,
+}
+
+impl<H> From<Message<H>> for MessageBag
+where
+ H: ConsensusHeader,
+{
+ fn from(_value: Message<H>) -> Self {
+ MessageBag::Void
+ }
+}
+
+pub(crate) mod header;
diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml
index 9bf2b9ad8..2240a10e2 100644
--- a/core/consensus/Cargo.toml
+++ b/core/consensus/Cargo.toml
@@ -29,3 +29,6 @@ repository = "https://github.com/apache/iggy"
readme = "../../../README.md"
[dependencies]
+clock = { path = "../clock" }
+
+
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/impls.rs
similarity index 89%
copy from core/consensus/src/lib.rs
copy to core/consensus/src/impls.rs
index c3241d3c7..be01c3b62 100644
--- a/core/consensus/src/lib.rs
+++ b/core/consensus/src/impls.rs
@@ -15,8 +15,4 @@
// specific language governing permissions and limitations
// under the License.
-pub trait Consensus {
- type RequestMessage;
- type ReplicateMessage;
- type AckMessage;
-}
+pub struct VsrConsensus;
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs
index c3241d3c7..0a02bc089 100644
--- a/core/consensus/src/lib.rs
+++ b/core/consensus/src/lib.rs
@@ -14,9 +14,25 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+use clock::Clock;
-pub trait Consensus {
- type RequestMessage;
- type ReplicateMessage;
+pub trait Project<U, T, C>
+where
+ C: Consensus<T>,
+ T: Clock,
+{
+ fn project(self, consensus: &C) -> U;
+}
+
+pub trait Consensus<C>
+where
+ Self: Sized,
+ C: Clock,
+{
+ type RequestMessage: Project<Self::ReplicateMessage, C, Self>;
+ type ReplicateMessage: Project<Self::AckMessage, C, Self>;
type AckMessage;
}
+
+mod impls;
+pub use impls::*;
diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml
index 87e578250..f026f763e 100644
--- a/core/metadata/Cargo.toml
+++ b/core/metadata/Cargo.toml
@@ -29,5 +29,7 @@ repository = "https://github.com/apache/iggy"
readme = "../../../README.md"
[dependencies]
+clock = { path = "../clock" }
consensus = { path = "../consensus" }
+tracing = { workspace = true }
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 659112f1e..f802d53f1 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -14,13 +14,17 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-use consensus::Consensus;
+use clock::Clock;
+use consensus::{Consensus, Project};
+use std::marker::PhantomData;
+use tracing::debug;
// TODO: Define a trait (probably in some external crate)
#[expect(unused)]
-trait Metadata<C>
+trait Metadata<C, T>
where
- C: Consensus,
+ C: Consensus<T>,
+ T: Clock,
{
fn on_request(&self, message: C::RequestMessage);
fn on_replicate(&self, message: C::ReplicateMessage);
@@ -28,19 +32,27 @@ where
}
#[expect(unused)]
-struct IggyMetadata<C, M, J, S> {
+struct IggyMetadata<C, M, J, S, T>
+where
+ C: Consensus<T>,
+ T: Clock,
+{
consensus: C,
mux_stm: M,
journal: J,
snapshot: S,
+
+ _t: PhantomData<T>,
}
-impl<C, M, J, S> Metadata<C> for IggyMetadata<C, M, J, S>
+impl<C, M, J, S, T> Metadata<C, T> for IggyMetadata<C, M, J, S, T>
where
- C: Consensus,
+ C: Consensus<T>,
+ T: Clock,
{
- fn on_request(&self, _message: C::RequestMessage) {
- todo!()
+ fn on_request(&self, message: C::RequestMessage) {
+ debug!("handling metadata request");
+ let _message = message.project(&self.consensus);
}
fn on_replicate(&self, _message: C::ReplicateMessage) {
@@ -51,3 +63,7 @@ where
todo!()
}
}
+
+// TODO: Hide with associated types all of those generics, so they are not leaking to the upper layer
+#[expect(unused)]
+pub trait MetadataHandle {}