You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/11/18 16:41:34 UTC

(iggy) branch master updated: feat(cluster): prepare consensus header setup and add clock trait (#2371)

This is an automated email from the ASF dual-hosted git repository.

gkoszyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git


The following commit(s) were added to refs/heads/master by this push:
     new 1151b95a6 feat(cluster): prepare consensus header setup and add clock trait (#2371)
1151b95a6 is described below

commit 1151b95a6ca808b8e9cbd7949a777cf7215b727d
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Tue Nov 18 17:41:29 2025 +0100

    feat(cluster): prepare consensus header setup and add clock trait (#2371)
---
 Cargo.lock                                         | 12 ++++
 Cargo.toml                                         |  1 +
 DEPENDENCIES.md                                    |  1 +
 core/{consensus => clock}/Cargo.toml               | 11 +---
 core/{consensus/src/lib.rs => clock/src/impls.rs}  | 15 +++--
 core/{consensus => clock}/src/lib.rs               | 10 ++--
 .../src/types/consensus/header.rs}                 |  9 ++-
 core/common/src/types/consensus/mod.rs             | 70 ++++++++++++++++++----
 core/consensus/Cargo.toml                          |  3 +
 core/consensus/src/{lib.rs => impls.rs}            |  6 +-
 core/consensus/src/lib.rs                          | 22 ++++++-
 core/metadata/Cargo.toml                           |  2 +
 core/metadata/src/impls/metadata.rs                | 32 +++++++---
 13 files changed, 146 insertions(+), 48 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5f7850cbc..3030058f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1413,6 +1413,13 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
 
+[[package]]
+name = "clock"
+version = "0.1.0"
+dependencies = [
+ "iggy_common",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.54"
@@ -1696,6 +1703,9 @@ dependencies = [
 [[package]]
 name = "consensus"
 version = "0.1.0"
+dependencies = [
+ "clock",
+]
 
 [[package]]
 name = "console"
@@ -5221,7 +5231,9 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
 name = "metadata"
 version = "0.1.0"
 dependencies = [
+ "clock",
  "consensus",
+ "tracing",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 53e3ae956..811ec468d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,6 +27,7 @@ members = [
     "core/bench/runner",
     "core/binary_protocol",
     "core/cli",
+    "core/clock",
     "core/common",
     "core/connectors/runtime",
     "core/connectors/sdk",
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 69c20d188..061e2ca25 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -122,6 +122,7 @@ clap_builder: 4.5.51, "Apache-2.0 OR MIT",
 clap_complete: 4.5.60, "Apache-2.0 OR MIT",
 clap_derive: 4.5.49, "Apache-2.0 OR MIT",
 clap_lex: 0.7.6, "Apache-2.0 OR MIT",
+clock: 0.1.0, "N/A",
 cmake: 0.1.54, "Apache-2.0 OR MIT",
 cobs: 0.3.0, "Apache-2.0 OR MIT",
 colorchoice: 1.0.4, "Apache-2.0 OR MIT",
diff --git a/core/consensus/Cargo.toml b/core/clock/Cargo.toml
similarity index 70%
copy from core/consensus/Cargo.toml
copy to core/clock/Cargo.toml
index 9bf2b9ad8..674f5bf8a 100644
--- a/core/consensus/Cargo.toml
+++ b/core/clock/Cargo.toml
@@ -16,16 +16,9 @@
 # under the License.
 
 [package]
-name = "consensus"
+name = "clock"
 version = "0.1.0"
-description = "Iggy consensus module"
 edition = "2024"
-license = "Apache-2.0"
-keywords = ["iggy", "messaging", "streaming"]
-categories = ["command-line-utilities", "database", "network-programming"]
-homepage = "https://iggy.apache.org"
-documentation = "https://iggy.apache.org/docs"
-repository = "https://github.com/apache/iggy"
-readme = "../../../README.md"
 
 [dependencies]
+iggy_common = { workspace = true }
diff --git a/core/consensus/src/lib.rs b/core/clock/src/impls.rs
similarity index 77%
copy from core/consensus/src/lib.rs
copy to core/clock/src/impls.rs
index c3241d3c7..6b34f289e 100644
--- a/core/consensus/src/lib.rs
+++ b/core/clock/src/impls.rs
@@ -15,8 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-pub trait Consensus {
-    type RequestMessage;
-    type ReplicateMessage;
-    type AckMessage;
+use crate::Clock;
+use iggy_common::IggyTimestamp;
+
+pub struct IggySystemClock;
+
+impl Clock for IggySystemClock {
+    type Realtime = IggyTimestamp;
+
+    fn realtime(&self) -> Self::Realtime {
+        IggyTimestamp::now()
+    }
 }
diff --git a/core/consensus/src/lib.rs b/core/clock/src/lib.rs
similarity index 86%
copy from core/consensus/src/lib.rs
copy to core/clock/src/lib.rs
index c3241d3c7..2223894ed 100644
--- a/core/consensus/src/lib.rs
+++ b/core/clock/src/lib.rs
@@ -15,8 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-pub trait Consensus {
-    type RequestMessage;
-    type ReplicateMessage;
-    type AckMessage;
+pub trait Clock {
+    type Realtime;
+    fn realtime(&self) -> Self::Realtime;
 }
+
+mod impls;
+pub use impls::IggySystemClock;
diff --git a/core/consensus/src/lib.rs b/core/common/src/types/consensus/header.rs
similarity index 89%
copy from core/consensus/src/lib.rs
copy to core/common/src/types/consensus/header.rs
index c3241d3c7..96ad1eaa0 100644
--- a/core/consensus/src/lib.rs
+++ b/core/common/src/types/consensus/header.rs
@@ -15,8 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-pub trait Consensus {
-    type RequestMessage;
-    type ReplicateMessage;
-    type AckMessage;
-}
+#[expect(unused)]
+pub struct Header {}
+
+pub trait ConsensusHeader {}
diff --git a/core/common/src/types/consensus/mod.rs b/core/common/src/types/consensus/mod.rs
index 6ecac7726..795695d12 100644
--- a/core/common/src/types/consensus/mod.rs
+++ b/core/common/src/types/consensus/mod.rs
@@ -17,6 +17,10 @@
 
 use std::marker::PhantomData;
 
+use bytes::Bytes;
+
+use crate::types::consensus::header::ConsensusHeader;
+
 // TODO: Header generic
 // TODO: We will have to impl something like this (NOT ONE TO ONE JUST A SKETCH) as we will use `bytemuck`:
 /*
@@ -40,14 +44,20 @@ pub trait Header: Sized {
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 #[repr(u8)]
 pub enum Command {
-    Reserved = 0,
-    Ping = 1,
-    Pong = 2,
-    Request = 3,
-    Prepare = 4,
-    PrepareOk = 5,
-    Reply = 6,
-    Commit = 7,
+    reserved = 0,
+
+    ping = 1,
+    pong = 2,
+
+    request = 5,
+    prepare = 6,
+    prepare_ok = 7,
+    reply = 8,
+    commit = 9,
+
+    start_view_change = 10,
+    do_view_change = 11,
+    start_view = 24,
     // ... etc
 }
 
@@ -60,6 +70,9 @@ pub struct GenericHeader {
     // ... other fields
 }
 
+// first 128 bytes GenericHeader (checksum, command, size, etc....)
+// second 128 bytes specific header (PrepareHeader, CommitHeader, etc....)
+
 impl Header for GenericHeader {
     const COMMAND: Command = Command::Reserved;
 
@@ -86,6 +99,7 @@ impl Header for PrepareHeader {
     fn size(&self) -> u32 { self.size }
     fn command(&self) -> Command { self.command }
     fn checksum(&self) -> u128 { self.checksum }
+}
 */
 
 // And then for Message impl
@@ -152,11 +166,47 @@ pub enum Operation {
     Consume = 101,
     Fetch = 102,
 }
+
+// Specific header types
+#[repr(C)]
+pub struct PrepareHeader {
+    checksum: u128,
+    command: Command,
+    size: u32,
+    // ... prepare-specific fields
+    view: u32,
+    op: u64,
+    commit: u64,
+
+    // second 128 bytes
+    operation: Operation,
+}
 */
 
 // Which will have an method that returns discriminator between Metadata and Partition requests
 
+// TODO: Fill this enum
 #[expect(unused)]
-pub struct Message<H> {
-    _phantom_header: PhantomData<H>,
+pub enum MessageBag {
+    Void,
 }
+
+#[expect(unused)]
+pub struct Message<H>
+where
+    H: ConsensusHeader,
+{
+    buffer: Bytes,
+    _h: PhantomData<H>,
+}
+
+impl<H> From<Message<H>> for MessageBag
+where
+    H: ConsensusHeader,
+{
+    fn from(_value: Message<H>) -> Self {
+        MessageBag::Void
+    }
+}
+
+pub(crate) mod header;
diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml
index 9bf2b9ad8..2240a10e2 100644
--- a/core/consensus/Cargo.toml
+++ b/core/consensus/Cargo.toml
@@ -29,3 +29,6 @@ repository = "https://github.com/apache/iggy"
 readme = "../../../README.md"
 
 [dependencies]
+clock = { path = "../clock" }
+
+
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/impls.rs
similarity index 89%
copy from core/consensus/src/lib.rs
copy to core/consensus/src/impls.rs
index c3241d3c7..be01c3b62 100644
--- a/core/consensus/src/lib.rs
+++ b/core/consensus/src/impls.rs
@@ -15,8 +15,4 @@
 // specific language governing permissions and limitations
 // under the License.
 
-pub trait Consensus {
-    type RequestMessage;
-    type ReplicateMessage;
-    type AckMessage;
-}
+pub struct VsrConsensus;
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs
index c3241d3c7..0a02bc089 100644
--- a/core/consensus/src/lib.rs
+++ b/core/consensus/src/lib.rs
@@ -14,9 +14,25 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+use clock::Clock;
 
-pub trait Consensus {
-    type RequestMessage;
-    type ReplicateMessage;
+pub trait Project<U, T, C>
+where
+    C: Consensus<T>,
+    T: Clock,
+{
+    fn project(self, consensus: &C) -> U;
+}
+
+pub trait Consensus<C>
+where
+    Self: Sized,
+    C: Clock,
+{
+    type RequestMessage: Project<Self::ReplicateMessage, C, Self>;
+    type ReplicateMessage: Project<Self::AckMessage, C, Self>;
     type AckMessage;
 }
+
+mod impls;
+pub use impls::*;
diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml
index 87e578250..f026f763e 100644
--- a/core/metadata/Cargo.toml
+++ b/core/metadata/Cargo.toml
@@ -29,5 +29,7 @@ repository = "https://github.com/apache/iggy"
 readme = "../../../README.md"
 
 [dependencies]
+clock = { path = "../clock" }
 consensus = { path = "../consensus" }
+tracing = { workspace = true }
 
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 659112f1e..f802d53f1 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -14,13 +14,17 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-use consensus::Consensus;
+use clock::Clock;
+use consensus::{Consensus, Project};
+use std::marker::PhantomData;
+use tracing::debug;
 
 // TODO: Define a trait (probably in some external crate)
 #[expect(unused)]
-trait Metadata<C>
+trait Metadata<C, T>
 where
-    C: Consensus,
+    C: Consensus<T>,
+    T: Clock,
 {
     fn on_request(&self, message: C::RequestMessage);
     fn on_replicate(&self, message: C::ReplicateMessage);
@@ -28,19 +32,27 @@ where
 }
 
 #[expect(unused)]
-struct IggyMetadata<C, M, J, S> {
+struct IggyMetadata<C, M, J, S, T>
+where
+    C: Consensus<T>,
+    T: Clock,
+{
     consensus: C,
     mux_stm: M,
     journal: J,
     snapshot: S,
+
+    _t: PhantomData<T>,
 }
 
-impl<C, M, J, S> Metadata<C> for IggyMetadata<C, M, J, S>
+impl<C, M, J, S, T> Metadata<C, T> for IggyMetadata<C, M, J, S, T>
 where
-    C: Consensus,
+    C: Consensus<T>,
+    T: Clock,
 {
-    fn on_request(&self, _message: C::RequestMessage) {
-        todo!()
+    fn on_request(&self, message: C::RequestMessage) {
+        debug!("handling metadata request");
+        let _message = message.project(&self.consensus);
     }
 
     fn on_replicate(&self, _message: C::ReplicateMessage) {
@@ -51,3 +63,7 @@ where
         todo!()
     }
 }
+
+// TODO: Hide with associated types all of those generics, so they are not leaking to the upper layer
+#[expect(unused)]
+pub trait MetadataHandle {}