You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/01/05 17:37:08 UTC
(iggy) 01/01: global-metadata
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch global-metadata
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit bff496981c017fc2eb17bee83e77964640acbdf8
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Jan 5 14:14:16 2026 +0100
global-metadata
---
Cargo.lock | 41 +-
DEPENDENCIES.md | 4 +
core/configs/server.toml | 2 +-
core/integration/src/test_server.rs | 6 +-
core/integration/tests/server/mod.rs | 6 +
core/integration/tests/streaming/mod.rs | 799 ++++++++++-
core/server/Cargo.toml | 2 +
.../create_consumer_group_handler.rs | 47 +-
.../delete_consumer_group_handler.rs | 29 +-
.../consumer_groups/get_consumer_group_handler.rs | 37 +-
.../consumer_groups/get_consumer_groups_handler.rs | 51 +-
.../handlers/messages/send_messages_handler.rs | 74 +-
.../partitions/create_partitions_handler.rs | 142 +-
.../partitions/delete_partitions_handler.rs | 146 +-
.../create_personal_access_token_handler.rs | 5 -
.../delete_personal_access_token_handler.rs | 7 -
.../handlers/segments/delete_segments_handler.rs | 17 +-
.../handlers/streams/create_stream_handler.rs | 18 +-
.../handlers/streams/delete_stream_handler.rs | 13 +-
.../binary/handlers/streams/get_stream_handler.rs | 24 +-
.../binary/handlers/streams/get_streams_handler.rs | 10 +-
.../handlers/streams/update_stream_handler.rs | 85 +-
.../binary/handlers/topics/create_topic_handler.rs | 66 +-
.../binary/handlers/topics/delete_topic_handler.rs | 20 +-
.../binary/handlers/topics/get_topic_handler.rs | 42 +-
.../binary/handlers/topics/get_topics_handler.rs | 23 +-
.../binary/handlers/topics/update_topic_handler.rs | 46 +-
.../handlers/users/change_password_handler.rs | 8 -
.../binary/handlers/users/create_user_handler.rs | 11 -
.../binary/handlers/users/delete_user_handler.rs | 3 -
.../handlers/users/update_permissions_handler.rs | 6 -
.../binary/handlers/users/update_user_handler.rs | 8 -
core/server/src/binary/mapper.rs | 152 +-
core/server/src/bootstrap.rs | 389 ++---
core/server/src/configs/system.rs | 9 +-
core/server/src/http/consumer_groups.rs | 196 ++-
core/server/src/http/consumer_offsets.rs | 6 +-
core/server/src/http/http_shard_wrapper.rs | 65 +-
core/server/src/http/mapper.rs | 464 +++---
core/server/src/http/partitions.rs | 4 +-
core/server/src/http/streams.rs | 71 +-
core/server/src/http/topics.rs | 176 +--
core/server/src/http/users.rs | 89 --
core/server/src/lib.rs | 2 +-
core/server/src/main.rs | 86 +-
core/server/src/metadata/consumer_group.rs | 70 +
core/server/src/metadata/consumer_offsets_store.rs | 127 ++
core/server/src/metadata/mod.rs | 55 +
core/server/src/metadata/partition.rs | 33 +
core/server/src/metadata/shared.rs | 1385 ++++++++++++++++++
core/server/src/metadata/snapshot.rs | 58 +
core/server/src/metadata/stream.rs | 55 +
core/server/src/metadata/topic.rs | 36 +
core/server/src/metadata/user.rs | 30 +
core/server/src/shard/builder.rs | 58 +-
core/server/src/shard/handlers.rs | 610 +++++---
core/server/src/shard/mod.rs | 133 +-
core/server/src/shard/namespace.rs | 12 +-
core/server/src/shard/partition_store.rs | 288 ++++
core/server/src/shard/system/clients.rs | 2 -
core/server/src/shard/system/consumer_groups.rs | 482 ++++---
core/server/src/shard/system/consumer_offsets.rs | 446 +++---
core/server/src/shard/system/messages.rs | 912 +++++++++++-
core/server/src/shard/system/partitions.rs | 513 +++++--
.../src/shard/system/personal_access_tokens.rs | 164 +--
core/server/src/shard/system/segments.rs | 166 ++-
core/server/src/shard/system/stats.rs | 47 +-
core/server/src/shard/system/streams.rs | 373 ++++-
core/server/src/shard/system/topics.rs | 629 ++++++--
core/server/src/shard/system/users.rs | 265 ++--
core/server/src/shard/system/utils.rs | 193 ++-
.../src/shard/tasks/periodic/message_cleaner.rs | 179 +--
.../src/shard/tasks/periodic/message_saver.rs | 82 +-
.../periodic/personal_access_token_cleaner.rs | 44 +-
core/server/src/shard/transmission/event.rs | 58 +-
core/server/src/shard/transmission/frame.rs | 21 +-
core/server/src/shard/transmission/message.rs | 20 +-
core/server/src/slab/consumer_groups.rs | 173 ---
core/server/src/slab/helpers.rs | 70 -
core/server/src/slab/mod.rs | 43 -
core/server/src/slab/partitions.rs | 243 ----
core/server/src/slab/streams.rs | 1497 --------------------
core/server/src/slab/topics.rs | 279 ----
core/server/src/slab/traits_ext.rs | 117 --
core/server/src/slab/users.rs | 207 ---
core/server/src/streaming/partitions/helpers.rs | 512 +------
core/server/src/streaming/partitions/partition.rs | 294 +---
core/server/src/streaming/stats/mod.rs | 15 +
core/server/src/streaming/streams/helpers.rs | 107 --
core/server/src/streaming/streams/mod.rs | 2 -
core/server/src/streaming/streams/storage.rs | 27 +-
core/server/src/streaming/streams/stream.rs | 221 ---
core/server/src/streaming/topics/consumer_group.rs | 243 ----
core/server/src/streaming/topics/helpers.rs | 306 +---
core/server/src/streaming/topics/mod.rs | 2 -
core/server/src/streaming/topics/storage.rs | 43 +-
core/server/src/streaming/topics/topic.rs | 371 -----
core/server/src/streaming/users/permissioner.rs | 112 +-
.../streaming/users/permissioner_rules/messages.rs | 110 +-
.../streaming/users/permissioner_rules/streams.rs | 32 +-
.../streaming/users/permissioner_rules/system.rs | 6 +-
.../streaming/users/permissioner_rules/topics.rs | 80 +-
.../streaming/users/permissioner_rules/users.rs | 24 +-
103 files changed, 8222 insertions(+), 7967 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index b37ab1d22..70a3766c9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1184,6 +1184,12 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "bitmaps"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6"
+
[[package]]
name = "bitvec"
version = "1.0.1"
@@ -1957,7 +1963,7 @@ dependencies = [
"iggy_common",
"message_bus",
"rand 0.9.2",
- "rand_xoshiro",
+ "rand_xoshiro 0.7.0",
]
[[package]]
@@ -4892,6 +4898,28 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "imbl"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc3be8d8cd36f33a46b1849f31f837c44d9fa87223baee3b4bd96b8f11df81eb"
+dependencies = [
+ "bitmaps",
+ "imbl-sized-chunks",
+ "rand_core 0.6.4",
+ "rand_xoshiro 0.6.0",
+ "version_check",
+]
+
+[[package]]
+name = "imbl-sized-chunks"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f4241005618a62f8d57b2febd02510fb96e0137304728543dfc5fd6f052c22d"
+dependencies = [
+ "bitmaps",
+]
+
[[package]]
name = "impl-more"
version = "0.1.9"
@@ -7230,6 +7258,15 @@ dependencies = [
"getrandom 0.3.4",
]
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "rand_xoshiro"
version = "0.7.0"
@@ -8250,6 +8287,7 @@ version = "0.6.1-edge.2"
dependencies = [
"ahash 0.8.12",
"anyhow",
+ "arc-swap",
"arcshift",
"argon2",
"async-channel",
@@ -8282,6 +8320,7 @@ dependencies = [
"human-repr",
"hwlocality",
"iggy_common",
+ "imbl",
"jsonwebtoken",
"lending-iterator",
"mimalloc",
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 88133f67a..fb6a579e6 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -98,6 +98,7 @@ bit-set: 0.8.0, "Apache-2.0 OR MIT",
bit-vec: 0.8.0, "Apache-2.0 OR MIT",
bitflags: 1.3.2, "Apache-2.0 OR MIT",
bitflags: 2.10.0, "Apache-2.0 OR MIT",
+bitmaps: 3.2.1, "MPL-2.0+",
bitvec: 1.0.1, "MIT",
blake2: 0.10.6, "Apache-2.0 OR MIT",
blake3: 1.8.2, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0",
@@ -410,6 +411,8 @@ iggy_connector_sdk: 0.1.1-edge.1, "Apache-2.0",
iggy_connector_stdout_sink: 0.1.0, "Apache-2.0",
iggy_examples: 0.0.5, "Apache-2.0",
ignore: 0.4.25, "MIT OR Unlicense",
+imbl: 3.0.0, "MPL-2.0+",
+imbl-sized-chunks: 0.1.3, "MPL-2.0+",
impl-more: 0.1.9, "Apache-2.0 OR MIT",
implicit-clone: 0.6.0, "Apache-2.0 OR MIT",
implicit-clone-derive: 0.1.2, "Apache-2.0 OR MIT",
@@ -638,6 +641,7 @@ rand_chacha: 0.3.1, "Apache-2.0 OR MIT",
rand_chacha: 0.9.0, "Apache-2.0 OR MIT",
rand_core: 0.6.4, "Apache-2.0 OR MIT",
rand_core: 0.9.3, "Apache-2.0 OR MIT",
+rand_xoshiro: 0.6.0, "Apache-2.0 OR MIT",
rand_xoshiro: 0.7.0, "Apache-2.0 OR MIT",
raw-cpuid: 11.6.0, "MIT",
rayon: 1.11.0, "Apache-2.0 OR MIT",
diff --git a/core/configs/server.toml b/core/configs/server.toml
index cec71ddc7..c2a521b87 100644
--- a/core/configs/server.toml
+++ b/core/configs/server.toml
@@ -152,7 +152,7 @@ enabled = true
address = "127.0.0.1:8090"
# Enable TCP socket migration across shards.
-socket_migration = true
+socket_migration = false
# Whether to use ipv4 or ipv6
ipv6 = false
diff --git a/core/integration/src/test_server.rs b/core/integration/src/test_server.rs
index fd718178e..0c43c56c8 100644
--- a/core/integration/src/test_server.rs
+++ b/core/integration/src/test_server.rs
@@ -286,6 +286,8 @@ impl TestServer {
/// Returns false if the process has exited (crashed or terminated).
pub fn is_running(&mut self) -> bool {
if let Some(child) = self.child_handle.as_mut() {
+ // try_wait returns Ok(Some(status)) if process has exited,
+ // Ok(None) if still running, Err on error
match child.try_wait() {
Ok(Some(_)) => false, // Process has exited
Ok(None) => true, // Still running
@@ -299,7 +301,9 @@ impl TestServer {
/// Assert that the server is still running. If it has crashed,
/// panic with server logs included for debugging.
pub fn assert_running(&mut self) {
- if let Some(Ok(Some(exit_status))) = self.child_handle.as_mut().map(|c| c.try_wait()) {
+ if let Some(child) = self.child_handle.as_mut()
+ && let Ok(Some(exit_status)) = child.try_wait()
+ {
let (stdout_content, stderr_content) = self.collect_logs();
panic!(
"Server process has crashed with exit status: {}\n\n\
diff --git a/core/integration/tests/server/mod.rs b/core/integration/tests/server/mod.rs
index f1a3524ea..6ca3099de 100644
--- a/core/integration/tests/server/mod.rs
+++ b/core/integration/tests/server/mod.rs
@@ -113,8 +113,11 @@ async fn run_scenario(transport: TransportProtocol, scenario: ScenarioFn) {
"15s".to_string(),
);
let test_server = TestServer::new(Some(extra_envs), true, None, IpAddrKind::V4);
+
+ // Wrap in Arc<Mutex> for shared access between scenario and monitor
let test_server = Arc::new(Mutex::new(test_server));
+ // Start the server
test_server.lock().unwrap().start();
let client_factory: Box<dyn ClientFactory> = {
@@ -142,6 +145,7 @@ async fn run_scenario(transport: TransportProtocol, scenario: ScenarioFn) {
}
};
+ // Server crash monitor task - checks every 100ms if server is still alive
let monitor_server = test_server.clone();
let crash_monitor = async move {
loop {
@@ -154,6 +158,7 @@ async fn run_scenario(transport: TransportProtocol, scenario: ScenarioFn) {
}
};
+ // Race the scenario against the crash monitor
tokio::select! {
biased;
@@ -167,6 +172,7 @@ async fn run_scenario(transport: TransportProtocol, scenario: ScenarioFn) {
}
result = std::panic::AssertUnwindSafe(scenario(&*client_factory)).catch_unwind() => {
+ // Scenario completed (or panicked), verify server is still healthy
test_server.lock().unwrap().assert_running();
// Re-raise any panic from the scenario
diff --git a/core/integration/tests/streaming/mod.rs b/core/integration/tests/streaming/mod.rs
index 6ad773647..3c5bc66a9 100644
--- a/core/integration/tests/streaming/mod.rs
+++ b/core/integration/tests/streaming/mod.rs
@@ -16,31 +16,678 @@
* under the License.
*/
-use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
+use iggy_common::sharding::IggyNamespace;
+use iggy_common::{
+ CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize,
+ PollingKind,
+};
use server::{
configs::system::SystemConfig,
- shard::{task_registry::TaskRegistry, transmission::connector::ShardConnector},
- slab::{streams::Streams, traits_ext::EntityMarker},
+ metadata::SharedMetadata,
+ shard::{
+ namespace::IggyFullNamespace,
+ partition_store::{PartitionData, PartitionDataStore},
+ system::messages::PollingArgs,
+ task_registry::TaskRegistry,
+ transmission::connector::ShardConnector,
+ },
streaming::{
- self,
- partitions::{partition, storage::create_partition_file_hierarchy},
- segments::{Segment, storage::create_segment_storage},
- streams::{storage::create_stream_file_hierarchy, stream},
- topics::{storage::create_topic_file_hierarchy, topic},
+ partitions::{
+ helpers::create_message_deduplicator,
+ journal::Journal,
+ partition::{ConsumerGroupOffsets, ConsumerOffsets},
+ storage::create_partition_file_hierarchy,
+ },
+ polling_consumer::PollingConsumer,
+ segments::{
+ IggyMessagesBatchMut, IggyMessagesBatchSet, Segment, storage::create_segment_storage,
+ },
+ streams::storage::create_stream_file_hierarchy,
+ topics::storage::create_topic_file_hierarchy,
+ traits::MainOps,
},
};
+use std::cell::RefCell;
use std::rc::Rc;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
mod common;
mod get_by_offset;
mod get_by_timestamp;
mod snapshot;
+/// Test harness that uses SharedMetadata + PartitionDataStore
+pub struct TestStreams {
+ pub shared_metadata: &'static SharedMetadata,
+ pub partition_store: RefCell<PartitionDataStore>,
+}
+
+impl TestStreams {
+ pub fn new(shared_metadata: &'static SharedMetadata) -> Self {
+ Self {
+ shared_metadata,
+ partition_store: RefCell::new(PartitionDataStore::new()),
+ }
+ }
+
+ /// Access partition data by Identifier-based stream/topic IDs.
+ /// Converts Identifiers to numeric IDs using SharedMetadata, then accesses partition_store.
+ pub fn with_partition_by_id<F, R>(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: usize,
+ f: F,
+ ) -> R
+ where
+ F: FnOnce(
+ (
+ &PartitionData,
+ usize,
+ usize,
+ &Arc<AtomicU64>,
+ &Arc<ConsumerOffsets>,
+ &Arc<ConsumerGroupOffsets>,
+ ),
+ ) -> R,
+ {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let store = self.partition_store.borrow();
+ let partition_data = store.get(&namespace).expect("Partition must exist");
+
+ f((
+ partition_data,
+ numeric_stream_id,
+ numeric_topic_id,
+ &partition_data.offset,
+ &partition_data.consumer_offsets,
+ &partition_data.consumer_group_offsets,
+ ))
+ }
+}
+
+impl MainOps for TestStreams {
+ type Namespace = IggyFullNamespace;
+ type PollingArgs = PollingArgs;
+ type Consumer = PollingConsumer;
+ type In = IggyMessagesBatchMut;
+ type Out = (
+ server::binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
+ IggyMessagesBatchSet,
+ );
+ type Error = IggyError;
+
+ async fn append_messages(
+ &self,
+ config: &SystemConfig,
+ _registry: &Rc<TaskRegistry>,
+ ns: &Self::Namespace,
+ mut batch: Self::In,
+ ) -> Result<(), Self::Error> {
+ // Resolve Identifier to numeric IDs
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(ns.stream_id())
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, ns.topic_id())
+ .expect("Topic must exist");
+ let partition_id = ns.partition_id();
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ if batch.count() == 0 {
+ return Ok(());
+ }
+
+ // Get necessary data from partition_store
+ let (current_offset, current_position, segment_start_offset, message_deduplicator) = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(&namespace)
+ .expect("partition_store: partition must exist");
+
+ let current_offset = if partition_data.should_increment_offset {
+ partition_data.offset.load(Ordering::Relaxed) + 1
+ } else {
+ 0
+ };
+
+ let segment = partition_data.log.active_segment();
+ let current_position = segment.current_position;
+ let segment_start_offset = segment.start_offset;
+ let message_deduplicator = partition_data.message_deduplicator.clone();
+
+ (
+ current_offset,
+ current_position,
+ segment_start_offset,
+ message_deduplicator,
+ )
+ };
+
+ // Prepare batch for persistence (outside the borrow)
+ batch
+ .prepare_for_persistence(
+ segment_start_offset,
+ current_offset,
+ current_position,
+ message_deduplicator.as_ref(),
+ )
+ .await;
+
+ // Append to journal
+ let (journal_messages_count, journal_size) = {
+ let mut store = self.partition_store.borrow_mut();
+ let partition_data = store
+ .get_mut(&namespace)
+ .expect("partition_store: partition must exist");
+
+ let segment = partition_data.log.active_segment_mut();
+
+ if segment.end_offset == 0 {
+ segment.start_timestamp = batch.first_timestamp().unwrap();
+ }
+
+ let batch_messages_size = batch.size();
+ let batch_messages_count = batch.count();
+
+ partition_data
+ .stats
+ .increment_size_bytes(batch_messages_size as u64);
+ partition_data
+ .stats
+ .increment_messages_count(batch_messages_count as u64);
+
+ segment.end_timestamp = batch.last_timestamp().unwrap();
+ segment.end_offset = batch.last_offset().unwrap();
+
+ let (journal_messages_count, journal_size) =
+ partition_data.log.journal_mut().append(batch)?;
+
+ let last_offset = if batch_messages_count == 0 {
+ current_offset
+ } else {
+ current_offset + batch_messages_count as u64 - 1
+ };
+
+ if partition_data.should_increment_offset {
+ partition_data.offset.store(last_offset, Ordering::Relaxed);
+ } else {
+ partition_data.should_increment_offset = true;
+ partition_data.offset.store(last_offset, Ordering::Relaxed);
+ }
+ partition_data.log.active_segment_mut().current_position += batch_messages_size;
+
+ (journal_messages_count, journal_size)
+ };
+
+ // Check if journal should be persisted
+ let unsaved_messages_count_exceeded =
+ journal_messages_count >= config.partition.messages_required_to_save;
+ let unsaved_messages_size_exceeded = journal_size
+ >= config
+ .partition
+ .size_of_messages_required_to_save
+ .as_bytes_u64() as u32;
+
+ let is_full = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(&namespace)
+ .expect("partition_store: partition must exist");
+ partition_data.log.active_segment().is_full()
+ };
+
+ // Persist if needed
+ if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded {
+ // Commit journal and persist to disk
+ let batches = {
+ let mut store = self.partition_store.borrow_mut();
+ let partition_data = store
+ .get_mut(&namespace)
+ .expect("partition_store: partition must exist");
+ let batches = partition_data.log.journal_mut().commit();
+ partition_data.log.ensure_indexes();
+ batches.append_indexes_to(partition_data.log.active_indexes_mut().unwrap());
+ batches
+ };
+
+ self.persist_messages_to_disk(&namespace, batches).await?;
+ }
+
+ Ok(())
+ }
+
+ async fn poll_messages(
+ &self,
+ ns: &Self::Namespace,
+ consumer: Self::Consumer,
+ args: Self::PollingArgs,
+ ) -> Result<Self::Out, Self::Error> {
+ use server::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
+
+ // Resolve Identifier to numeric IDs
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(ns.stream_id())
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, ns.topic_id())
+ .expect("Topic must exist");
+ let partition_id = ns.partition_id();
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let count = args.count;
+ let strategy = args.strategy;
+ let value = strategy.value;
+
+ // Phase 1: Extract metadata and determine what we need
+ let (metadata, start_offset, _current_offset) = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(&namespace)
+ .expect("partition_store: partition must exist for poll");
+
+ let current_offset = partition_data.offset.load(Ordering::Relaxed);
+ let metadata = IggyPollMetadata::new(partition_id as u32, current_offset);
+
+ // Calculate start offset based on polling kind
+ let start_offset = match strategy.kind {
+ PollingKind::Offset => {
+ let offset = value;
+ if offset > current_offset {
+ return Ok((metadata, IggyMessagesBatchSet::empty()));
+ }
+ offset
+ }
+ PollingKind::First => partition_data
+ .log
+ .segments()
+ .first()
+ .map(|segment| segment.start_offset)
+ .unwrap_or(0),
+ PollingKind::Last => {
+ let mut requested_count = count as u64;
+ if requested_count > current_offset + 1 {
+ requested_count = current_offset + 1;
+ }
+ 1 + current_offset - requested_count
+ }
+ PollingKind::Next => {
+ // Get stored offset if exists (None means first poll ever)
+ let stored_offset = match consumer {
+ PollingConsumer::Consumer(id, _) => partition_data
+ .consumer_offsets
+ .pin()
+ .get(&id)
+ .map(|item| item.offset.load(Ordering::Relaxed)),
+ PollingConsumer::ConsumerGroup(cg_id, _) => partition_data
+ .consumer_group_offsets
+ .pin()
+ .get(&cg_id)
+ .map(|item| item.offset.load(Ordering::Relaxed)),
+ };
+ match stored_offset {
+ Some(offset) => offset + 1,
+ None => partition_data
+ .log
+ .segments()
+ .first()
+ .map(|segment| segment.start_offset)
+ .unwrap_or(0),
+ }
+ }
+ PollingKind::Timestamp => partition_data
+ .log
+ .segments()
+ .first()
+ .map(|segment| segment.start_offset)
+ .unwrap_or(0),
+ };
+
+ if start_offset > current_offset || count == 0 {
+ return Ok((metadata, IggyMessagesBatchSet::empty()));
+ }
+
+ (metadata, start_offset, current_offset)
+ };
+
+ // Phase 2: Try to get messages from journal first
+ let journal_result = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(&namespace)
+ .expect("partition_store: partition must exist for poll");
+
+ let journal = partition_data.log.journal();
+ let journal_inner = journal.inner();
+ let is_journal_empty = journal.is_empty();
+ let journal_first_offset = journal_inner.base_offset;
+ let journal_last_offset = journal_inner.current_offset;
+
+ let end_offset = start_offset + (count - 1).max(1) as u64;
+
+ // Case 1: All messages are in journal
+ if !is_journal_empty
+ && start_offset >= journal_first_offset
+ && end_offset <= journal_last_offset
+ {
+ let batches = journal.get(|batches| batches.get_by_offset(start_offset, count));
+ Some(batches)
+ } else {
+ None
+ }
+ };
+
+ if let Some(batches) = journal_result {
+ return Ok((metadata, batches));
+ }
+
+ // Phase 3: Load from disk
+ let batches = self
+ .load_messages_from_disk(&namespace, start_offset, count)
+ .await?;
+
+ Ok((metadata, batches))
+ }
+}
+
+impl TestStreams {
+ /// Persist messages to disk using partition_store's storage.
+ async fn persist_messages_to_disk(
+ &self,
+ namespace: &IggyNamespace,
+ batches: IggyMessagesBatchSet,
+ ) -> Result<u32, IggyError> {
+ let batch_count = batches.count();
+
+ if batch_count == 0 {
+ return Ok(0);
+ }
+
+ // Extract storage from partition_store
+ let (messages_writer, index_writer, has_segments) = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+
+ if !partition_data.log.has_segments() {
+ return Ok(0);
+ }
+
+ let messages_writer = partition_data
+ .log
+ .active_storage()
+ .messages_writer
+ .as_ref()
+ .expect("Messages writer not initialized")
+ .clone();
+ let index_writer = partition_data
+ .log
+ .active_storage()
+ .index_writer
+ .as_ref()
+ .expect("Index writer not initialized")
+ .clone();
+ (messages_writer, index_writer, true)
+ };
+
+ if !has_segments {
+ return Ok(0);
+ }
+
+ // Lock and save messages
+ let guard = messages_writer.lock.lock().await;
+ let saved = messages_writer.as_ref().save_batch_set(batches).await?;
+
+ // Get unsaved indexes from partition_store
+ let unsaved_indexes_slice = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+ partition_data.log.active_indexes().unwrap().unsaved_slice()
+ };
+
+ // Save indexes
+ index_writer
+ .as_ref()
+ .save_indexes(unsaved_indexes_slice)
+ .await?;
+
+ // Update index and increment segment stats
+ {
+ let mut store = self.partition_store.borrow_mut();
+ let partition_data = store
+ .get_mut(namespace)
+ .expect("partition_store: partition must exist");
+
+ let indexes = partition_data.log.active_indexes_mut().unwrap();
+ indexes.mark_saved();
+
+ let segment = partition_data.log.active_segment_mut();
+ segment.size =
+ iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64());
+ }
+
+ drop(guard);
+ Ok(batch_count)
+ }
+
+ /// Load messages from disk using partition_store's storage readers.
+ async fn load_messages_from_disk(
+ &self,
+ namespace: &IggyNamespace,
+ start_offset: u64,
+ count: u32,
+ ) -> Result<IggyMessagesBatchSet, IggyError> {
+ if count == 0 {
+ return Ok(IggyMessagesBatchSet::empty());
+ }
+
+ // Get segment range containing the requested offset
+ let segment_range = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+
+ let segments = partition_data.log.segments();
+ if segments.is_empty() {
+ return Ok(IggyMessagesBatchSet::empty());
+ }
+
+ let start = segments
+ .iter()
+ .rposition(|segment| segment.start_offset <= start_offset)
+ .unwrap_or(0);
+ let end = segments.len();
+ start..end
+ };
+
+ let mut remaining_count = count;
+ let mut batches = IggyMessagesBatchSet::empty();
+ let mut current_offset = start_offset;
+
+ for idx in segment_range {
+ if remaining_count == 0 {
+ break;
+ }
+
+ // Get segment info
+ let (segment_start_offset, segment_end_offset) = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+
+ let segment = &partition_data.log.segments()[idx];
+ (segment.start_offset, segment.end_offset)
+ };
+
+ let offset = if current_offset < segment_start_offset {
+ segment_start_offset
+ } else {
+ current_offset
+ };
+
+ let mut end_offset = offset + (remaining_count - 1).max(1) as u64;
+ if end_offset > segment_end_offset {
+ end_offset = segment_end_offset;
+ }
+
+ let messages = self
+ .load_segment_messages(
+ namespace,
+ idx,
+ offset,
+ end_offset,
+ remaining_count,
+ segment_start_offset,
+ )
+ .await?;
+
+ let loaded_count = messages.count();
+ if loaded_count > 0 {
+ batches.add_batch_set(messages);
+ remaining_count = remaining_count.saturating_sub(loaded_count);
+ current_offset = end_offset + 1;
+ } else {
+ break;
+ }
+ }
+
+ Ok(batches)
+ }
+
+ /// Load messages from a specific segment using partition_store.
+ async fn load_segment_messages(
+ &self,
+ namespace: &IggyNamespace,
+ idx: usize,
+ start_offset: u64,
+ end_offset: u64,
+ count: u32,
+ segment_start_offset: u64,
+ ) -> Result<IggyMessagesBatchSet, IggyError> {
+ let relative_start_offset = (start_offset - segment_start_offset) as u32;
+
+ // Check journal first for this segment's data
+ let journal_data = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+
+ let journal = partition_data.log.journal();
+ let is_journal_empty = journal.is_empty();
+ let journal_inner = journal.inner();
+ let journal_first_offset = journal_inner.base_offset;
+ let journal_last_offset = journal_inner.current_offset;
+
+ // Case 1: All messages are in journal
+ if !is_journal_empty
+ && start_offset >= journal_first_offset
+ && end_offset <= journal_last_offset
+ {
+ Some(journal.get(|batches| batches.get_by_offset(start_offset, count)))
+ } else {
+ None
+ }
+ };
+
+ if let Some(batches) = journal_data {
+ return Ok(batches);
+ }
+
+ // Case 2: Load from disk
+ let (index_reader, messages_reader, indexes) = {
+ let store = self.partition_store.borrow();
+ let partition_data = store
+ .get(namespace)
+ .expect("partition_store: partition must exist");
+
+ let storages = partition_data.log.storages();
+ if idx >= storages.len() {
+ return Ok(IggyMessagesBatchSet::empty());
+ }
+
+ let index_reader = storages[idx]
+ .index_reader
+ .as_ref()
+ .expect("Index reader not initialized")
+ .clone();
+ let messages_reader = storages[idx]
+ .messages_reader
+ .as_ref()
+ .expect("Messages reader not initialized")
+ .clone();
+ let indexes_vec = partition_data.log.indexes();
+ let indexes = indexes_vec
+ .get(idx)
+ .and_then(|opt| opt.as_ref())
+ .map(|indexes| {
+ indexes
+ .slice_by_offset(relative_start_offset, count)
+ .unwrap_or_default()
+ });
+ (index_reader, messages_reader, indexes)
+ };
+
+ // Get indexes: prefer in-memory, fallback to disk
+ let indexes_to_read = if let Some(indexes) = indexes {
+ if !indexes.is_empty() {
+ Some(indexes)
+ } else {
+ index_reader
+ .as_ref()
+ .load_from_disk_by_offset(relative_start_offset, count)
+ .await?
+ }
+ } else {
+ index_reader
+ .as_ref()
+ .load_from_disk_by_offset(relative_start_offset, count)
+ .await?
+ };
+
+ if indexes_to_read.is_none() {
+ return Ok(IggyMessagesBatchSet::empty());
+ }
+
+ let indexes_to_read = indexes_to_read.unwrap();
+ let batch = messages_reader
+ .as_ref()
+ .load_messages_from_disk(indexes_to_read)
+ .await?;
+
+ batch.validate_checksums_and_offsets(start_offset)?;
+
+ Ok(IggyMessagesBatchSet::from(batch))
+ }
+}
+
struct BootstrapResult {
- streams: Streams,
+ streams: TestStreams,
stream_id: Identifier,
topic_id: Identifier,
partition_id: usize,
+ #[allow(dead_code)]
+ namespace: IggyNamespace,
task_registry: Rc<TaskRegistry>,
}
@@ -54,79 +701,103 @@ async fn bootstrap_test_environment(
let topic_size = MaxTopicSize::Unlimited;
let partitions_count = 1;
- let streams = Streams::default();
- // Create stream together with its dirs
- let stream = stream::create_and_insert_stream_mem(&streams, stream_name);
- create_stream_file_hierarchy(stream.id(), config).await?;
- // Create topic together with its dirs
- let stream_id = Identifier::numeric(stream.id() as u32).unwrap();
- let parent_stats = streams.with_stream_by_id(&stream_id, |(_, stats)| stats.clone());
+ // Create SharedMetadata (leaked for 'static lifetime in tests)
+ let shared_metadata: &'static SharedMetadata = Box::leak(Box::new(SharedMetadata::default()));
+
+ // Create stream in SharedMetadata
+ let stream_id_num: usize = 1;
+ let _stream_stats = shared_metadata.register_stream(
+ stream_id_num,
+ Arc::from(stream_name.as_str()),
+ IggyTimestamp::now(),
+ );
+ create_stream_file_hierarchy(stream_id_num, config).await?;
+
+ // Create topic in SharedMetadata
+ let topic_id_num: usize = 1;
let message_expiry = config.resolve_message_expiry(topic_expiry);
let max_topic_size = config.resolve_max_topic_size(topic_size)?;
-
- let topic = topic::create_and_insert_topics_mem(
- &streams,
- &stream_id,
- topic_name,
- 1,
+ let _topic_stats = shared_metadata.register_topic(
+ stream_id_num,
+ topic_id_num,
+ Arc::from(topic_name.as_str()),
+ IggyTimestamp::now(),
message_expiry,
CompressionAlgorithm::default(),
max_topic_size,
- parent_stats,
+ 1, // replication_factor
+ partitions_count,
);
- create_topic_file_hierarchy(stream.id(), topic.id(), config).await?;
- // Create partition together with its dirs
- let topic_id = Identifier::numeric(topic.id() as u32).unwrap();
- let parent_stats = streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- streaming::topics::helpers::get_stats(),
+ create_topic_file_hierarchy(stream_id_num, topic_id_num, config).await?;
+
+ // Create partition in SharedMetadata
+ let partition_id: usize = 0;
+ let namespace = IggyNamespace::new(stream_id_num, topic_id_num, partition_id);
+ let partition_stats = shared_metadata.register_partition(namespace, IggyTimestamp::now());
+ create_partition_file_hierarchy(stream_id_num, topic_id_num, partition_id, config).await?;
+
+ // Create TestStreams with partition data
+ let streams = TestStreams::new(shared_metadata);
+
+ // Initialize partition data in partition_store
+ let start_offset = 0u64;
+ let segment = Segment::new(
+ start_offset,
+ config.segment.size,
+ config.segment.message_expiry,
);
- let partitions = partition::create_and_insert_partitions_mem(
- &streams,
- &stream_id,
- &topic_id,
- parent_stats,
- partitions_count,
+ let messages_size = 0;
+ let indexes_size = 0;
+ let storage = create_segment_storage(
config,
+ stream_id_num,
+ topic_id_num,
+ partition_id,
+ messages_size,
+ indexes_size,
+ start_offset,
+ )
+ .await?;
+
+ // Create partition data with the log
+ let consumer_offsets = Arc::new(ConsumerOffsets::with_capacity(0));
+ let consumer_group_offsets = Arc::new(ConsumerGroupOffsets::with_capacity(0));
+ let message_deduplicator = create_message_deduplicator(config).map(Arc::new);
+ let current_offset = Arc::new(AtomicU64::new(0));
+
+ let mut partition_data = PartitionData::new(
+ partition_stats,
+ current_offset,
+ consumer_offsets,
+ consumer_group_offsets,
+ message_deduplicator,
+ IggyTimestamp::now(),
+ 0, // creation_version
+ false, // should_increment_offset
);
- for partition in partitions {
- create_partition_file_hierarchy(stream.id(), topic.id(), partition.id(), config).await?;
-
- // Open the log
- let start_offset = 0;
- let segment = Segment::new(
- start_offset,
- config.segment.size,
- config.segment.message_expiry,
- );
- let messages_size = 0;
- let indexes_size = 0;
- let storage = create_segment_storage(
- config,
- stream.id(),
- topic.id(),
- partition.id(),
- messages_size,
- indexes_size,
- start_offset,
- )
- .await?;
-
- streams.with_partition_by_id_mut(&stream_id, &topic_id, partition.id(), |(.., log)| {
- log.add_persisted_segment(segment, storage);
- });
- }
+
+ // Add the segment to the log
+ partition_data.log.add_persisted_segment(segment, storage);
+
+ // Insert into partition store
+ streams
+ .partition_store
+ .borrow_mut()
+ .insert(namespace, partition_data);
// Create a test task registry with dummy stop sender from ShardConnector
let connector: ShardConnector<()> = ShardConnector::new(shard_id);
let task_registry = Rc::new(TaskRegistry::new(shard_id, vec![connector.stop_sender]));
+ let stream_id = Identifier::numeric(stream_id_num as u32).unwrap();
+ let topic_id = Identifier::numeric(topic_id_num as u32).unwrap();
+
Ok(BootstrapResult {
streams,
stream_id,
topic_id,
- partition_id: 0,
+ partition_id,
+ namespace,
task_registry,
})
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 41c7d88b0..b51f1b6c2 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -40,6 +40,7 @@ iggy-web = ["dep:rust-embed", "dep:mime_guess"]
[dependencies]
ahash = { workspace = true }
anyhow = { workspace = true }
+arc-swap = "1.7"
arcshift = "0.4.2"
argon2 = { workspace = true }
async-channel = { workspace = true }
@@ -72,6 +73,7 @@ hash32 = "1.0.0"
human-repr = { workspace = true }
iggy_common = { workspace = true }
+imbl = "3"
jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] }
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 192d867f5..5561c9dc6 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -24,14 +24,14 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::session::Session;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_consumer_group::CreateConsumerGroup;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -49,18 +49,17 @@ impl ServerCommandHandler for CreateConsumerGroup {
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
- let cg = shard.create_consumer_group(
+ let cg_id = shard.create_consumer_group(
session,
&self.stream_id,
&self.topic_id,
self.name.clone(),
)?;
- let cg_id = cg.id();
let event = ShardEvent::CreatedConsumerGroup {
stream_id: self.stream_id.clone(),
topic_id: self.topic_id.clone(),
- cg,
+ cg_id,
};
shard.broadcast_event_to_all_shards(event).await?;
@@ -68,25 +67,37 @@ impl ServerCommandHandler for CreateConsumerGroup {
let topic_id = self.topic_id.clone();
shard
.state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
- group_id: cg_id as u32,
- command: self
- }),
- )
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
+ group_id: cg_id as u32,
+ command: self,
+ }),
+ )
.await
.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to apply create consumer group for stream_id: {stream_id}, topic_id: {topic_id}, group_id: {cg_id}, session: {session}"
)
})?;
- let response = shard.streams.with_consumer_group_by_id(
- &stream_id,
- &topic_id,
- &Identifier::numeric(cg_id as u32).unwrap(),
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+
+ // Get consumer group from SharedMetadata (source of truth, works on all shards)
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ let key = (numeric_stream_id, numeric_topic_id, cg_id);
+ let metadata = shard.shared_metadata.load();
+ let response = metadata
+ .consumer_groups
+ .get(&key)
+ .map(mapper::map_consumer_group_from_meta)
+ .expect("Consumer group must exist after creation");
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
index 05c3cd9de..5f20d6497 100644
--- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
@@ -24,7 +24,7 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::streaming::polling_consumer::ConsumerGroupId;
use crate::streaming::session::Session;
@@ -49,33 +49,20 @@ impl ServerCommandHandler for DeleteConsumerGroup {
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
- let cg = shard.delete_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id).error(|e: &IggyError| {
+ let cg_meta = shard.delete_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id).error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {} for topic with ID: {} in stream with ID: {} for session: {}",
self.group_id, self.topic_id, self.stream_id, session
)
})?;
- let cg_id = cg.id();
- let partition_ids = cg.partitions();
-
- // Remove all consumer group members from ClientManager using helper functions to resolve identifiers
- let stream_id_usize = shard.streams.with_stream_by_id(
- &self.stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
- let topic_id_usize = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
+ let cg_id = cg_meta.id;
- // Get members from the deleted consumer group and make them leave
- let slab = cg.members().inner().shared_get();
- for (_, member) in slab.iter() {
+ // Make members leave the consumer group
+ for member in cg_meta.members.iter() {
if let Err(err) = shard.client_manager.leave_consumer_group(
member.client_id,
- stream_id_usize,
- topic_id_usize,
+ cg_meta.stream_id,
+ cg_meta.topic_id,
cg_id,
) {
tracing::warn!(
@@ -92,7 +79,7 @@ impl ServerCommandHandler for DeleteConsumerGroup {
cg_id_spez,
&self.stream_id,
&self.topic_id,
- partition_ids,
+ &cg_meta.partitions,
).await.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
index 7156a4d53..d57e4ea32 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -52,17 +51,18 @@ impl ServerCommandHandler for GetConsumerGroup {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
+
+ // Use SharedMetadata for ID resolution
let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ .shared_metadata
+ .get_stream_id(&self.stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &self.topic_id)
+ .expect("Topic must exist");
let has_permission = shard
.permissioner
- .borrow()
.get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.is_ok();
if !has_permission {
@@ -70,12 +70,19 @@ impl ServerCommandHandler for GetConsumerGroup {
return Ok(HandlerResult::Finished);
}
- let consumer_group = shard.streams.with_consumer_group_by_id(
- &self.stream_id,
- &self.topic_id,
- &self.group_id,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ // Get consumer group from SharedMetadata (source of truth)
+ let numeric_group_id = shard
+ .shared_metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &self.group_id)
+ .expect("Consumer group must exist");
+
+ let key = (numeric_stream_id, numeric_topic_id, numeric_group_id);
+ let metadata = shard.shared_metadata.load();
+ let consumer_group = metadata
+ .consumer_groups
+ .get(&key)
+ .map(mapper::map_consumer_group_from_meta)
+ .expect("Consumer group must exist");
sender.send_ok_response(&consumer_group).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
index d59f33c97..99e1c96e9 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
@@ -20,11 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -47,29 +44,45 @@ impl ServerCommandHandler for GetConsumerGroups {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?;
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
+
+ // Use SharedMetadata for ID resolution
let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
- shard.permissioner.borrow().get_consumer_groups(
+ .shared_metadata
+ .get_stream_id(&self.stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &self.topic_id)
+ .expect("Topic must exist");
+ shard.permissioner.get_consumer_groups(
session.get_user_id(),
numeric_stream_id,
numeric_topic_id,
)?;
+ // Get consumer groups from SharedMetadata (source of truth)
+ let metadata = shard.shared_metadata.load();
+ let topic_key = (numeric_stream_id, numeric_topic_id);
+
let consumer_groups =
- shard
- .streams
- .with_consumer_groups(&self.stream_id, &self.topic_id, |cgs| {
- cgs.with_components(|cgs| {
- let (roots, members) = cgs.into_components();
- mapper::map_consumer_groups(roots, members)
- })
- });
+ if let Some(group_ids) = metadata.consumer_groups_by_topic.get(&topic_key) {
+ use bytes::{BufMut, BytesMut};
+ let mut bytes = BytesMut::new();
+ for &group_id in group_ids.iter() {
+ let key = (numeric_stream_id, numeric_topic_id, group_id);
+ if let Some(cg_meta) = metadata.consumer_groups.get(&key) {
+ // Serialize consumer group header (without members for list view)
+ bytes.put_u32_le(cg_meta.id as u32);
+ bytes.put_u32_le(cg_meta.partitions.len() as u32);
+ bytes.put_u32_le(cg_meta.members.len() as u32);
+ bytes.put_u8(cg_meta.name.len() as u8);
+ bytes.put_slice(cg_meta.name.as_bytes());
+ }
+ }
+ bytes.freeze()
+ } else {
+ bytes::Bytes::new()
+ };
sender.send_ok_response(&consumer_groups).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 375848799..2648a9a06 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -21,7 +21,7 @@ use crate::shard::IggyShard;
use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
+use crate::streaming::topics;
use anyhow::Result;
use compio::buf::{IntoInner as _, IoBuf};
use iggy_common::Identifier;
@@ -112,42 +112,44 @@ impl ServerCommandHandler for SendMessages {
shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?;
+ // Use SharedMetadata for ID resolution
let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
-
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
-
- // TODO(tungtose): dry this code && get partition_id below have a side effect
- let partition_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- |(root, auxilary, ..)| match self.partitioning.kind {
- PartitioningKind::Balanced => {
- let upperbound = root.partitions().len();
- let pid = auxilary.get_next_partition_id(upperbound);
- Ok(pid)
- }
- PartitioningKind::PartitionId => Ok(u32::from_le_bytes(
- self.partitioning.value[..self.partitioning.length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- ) as usize),
- PartitioningKind::MessagesKey => {
- let upperbound = root.partitions().len();
- Ok(
- topics::helpers::calculate_partition_id_by_messages_key_hash(
- upperbound,
- &self.partitioning.value,
- ),
- )
- }
- },
- )?;
+ .shared_metadata
+ .get_stream_id(&self.stream_id)
+ .expect("Stream must exist");
+
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &self.topic_id)
+ .expect("Topic must exist");
+
+ // Get partition count from SharedMetadata (works on all shards)
+ let partitions_count = shard
+ .shared_metadata
+ .partitions_count(numeric_stream_id, numeric_topic_id);
+
+ // Calculate partition_id using SharedMetadata (no slab access)
+ let partition_id = match self.partitioning.kind {
+ PartitioningKind::Balanced => {
+ // Use topic's round-robin counter from SharedMetadata
+ shard.shared_metadata.get_next_partition_id(
+ numeric_stream_id,
+ numeric_topic_id,
+ partitions_count,
+ )
+ }
+ PartitioningKind::PartitionId => u32::from_le_bytes(
+ self.partitioning.value[..self.partitioning.length as usize]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ) as usize,
+ PartitioningKind::MessagesKey => {
+ topics::helpers::calculate_partition_id_by_messages_key_hash(
+ partitions_count,
+ &self.partitioning.value,
+ )
+ }
+ };
let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
let user_id = session.get_user_id();
diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
index 5c1930a8a..22387670e 100644
--- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
@@ -21,17 +21,17 @@ use crate::binary::command::{
};
use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_partitions::CreatePartitions;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,52 +50,98 @@ impl ServerCommandHandler for CreatePartitions {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
- // Acquire partition lock to serialize filesystem operations
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- let partitions = shard
- .create_partitions(
- session,
- &self.stream_id,
- &self.topic_id,
- self.partitions_count,
- )
- .await?;
- let partition_ids = partitions.iter().map(|p| p.id()).collect::<Vec<_>>();
- let event = ShardEvent::CreatedPartitions {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions,
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::CreatePartitions {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partitions_count: self.partitions_count,
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- shard.streams.with_topic_by_id_mut(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::rebalance_consumer_groups(&partition_ids),
- );
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::CreatePartitions {
+ stream_id,
+ topic_id,
+ partitions_count,
+ ..
+ } = payload
+ {
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ // Get numeric IDs BEFORE create (for rebalance operation)
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ let partition_infos = shard
+ .create_partitions(session, &stream_id, &topic_id, partitions_count)
+ .await?;
+ let partition_ids = partition_infos.iter().map(|p| p.id).collect::<Vec<_>>();
+
+ let event = crate::shard::transmission::event::ShardEvent::CreatedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions: partition_infos,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ // Rebalance consumer groups using SharedMetadata
+ shard.shared_metadata.rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ &partition_ids,
+ );
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::CreatePartitions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {numeric_stream_id}, topic_id: {numeric_topic_id}, session: {session}"
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected a CreatePartitions request inside of CreatePartitions handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::CreatePartitionsResponse(_partitions) => {
+ // Consumer group rebalancing is handled via the CreatedPartitions event
+ // broadcast in the shard-0 handler, so we don't need to do it here.
+ // Apply state for WAL replay.
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::CreatePartitions(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a CreatePartitionsResponse inside of CreatePartitions handler, impossible state"
+ ),
+ },
+ }
- let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
- let topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreatePartitions(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}"
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
index 9090ec2c8..de476c779 100644
--- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
@@ -23,13 +23,16 @@ use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_partitions::DeletePartitions;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -47,52 +50,107 @@ impl ServerCommandHandler for DeletePartitions {
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
-
- // Acquire partition lock to serialize filesystem operations
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
- let deleted_partition_ids = shard
- .delete_partitions(
- session,
- &self.stream_id,
- &self.topic_id,
- self.partitions_count,
- )
- .await?;
- let event = ShardEvent::DeletedPartitions {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions_count: self.partitions_count,
- partition_ids: deleted_partition_ids,
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::DeletePartitions {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partitions_count: self.partitions_count,
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- let remaining_partition_ids = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::get_partition_ids(),
- );
- shard.streams.with_topic_by_id_mut(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::rebalance_consumer_groups(&remaining_partition_ids),
- );
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::DeletePartitions {
+ stream_id,
+ topic_id,
+ partitions_count,
+ ..
+ } = payload
+ {
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ // Get numeric IDs BEFORE delete
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ let deleted_partition_ids = shard
+ .delete_partitions(session, &stream_id, &topic_id, partitions_count)
+ .await?;
+
+ let event = crate::shard::transmission::event::ShardEvent::DeletedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions_count,
+ partition_ids: deleted_partition_ids,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ // Rebalance consumer groups using SharedMetadata
+ let remaining_partition_ids: Vec<_> = shard
+ .shared_metadata
+ .load()
+ .partitions_by_topic
+ .get(&(numeric_stream_id, numeric_topic_id))
+ .map(|set| set.iter().copied().collect())
+ .unwrap_or_default();
+ shard.shared_metadata.rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ &remaining_partition_ids,
+ );
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::DeletePartitions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {}, topic_id: {}, session: {session}",
+ stream_id, topic_id
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected a DeletePartitions request inside of DeletePartitions handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::DeletePartitionsResponse(_partition_ids) => {
+ // Consumer group rebalancing is handled via the DeletedPartitions event
+ // broadcast in the shard-0 handler, so we don't need to do it here.
+ // Apply state for WAL replay.
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::DeletePartitions(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a DeletePartitionsResponse inside of DeletePartitions handler, impossible state"
+ ),
+ },
+ }
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::DeletePartitions(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}"
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
index 50b896fb0..ac9e5a886 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
@@ -24,7 +24,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::state::models::CreatePersonalAccessTokenWithHash;
use crate::streaming::session::Session;
@@ -60,10 +59,6 @@ impl ServerCommandHandler for CreatePersonalAccessToken {
})?;
let bytes = mapper::map_raw_pat(&token);
let hash = personal_access_token.token.to_string();
- let event = ShardEvent::CreatedPersonalAccessToken {
- personal_access_token: personal_access_token.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
shard
.state
diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
index bf89c70ca..77d9c2a61 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
@@ -53,13 +53,6 @@ impl ServerCommandHandler for DeletePersonalAccessToken {
"{COMPONENT} (error: {e}) - failed to delete personal access token with name: {token_name}, session: {session}"
)})?;
- // Broadcast the event to other shards
- let event = crate::shard::transmission::event::ShardEvent::DeletedPersonalAccessToken {
- user_id: session.get_user_id(),
- name: self.name.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
shard
.state
.apply(
diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
index a4eb20654..6a36d4b2c 100644
--- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs
+++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
@@ -27,7 +27,6 @@ use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
use crate::state::command::EntryCommand;
-use crate::streaming;
use crate::streaming::session::Session;
use anyhow::Result;
use err_trail::ErrContext;
@@ -62,16 +61,16 @@ impl ServerCommandHandler for DeleteSegments {
shard.ensure_topic_exists(&stream_id, &topic_id)?;
shard.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
- // Get numeric IDs for namespace
+ // Use SharedMetadata for ID resolution
let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&stream_id, streaming::streams::helpers::get_stream_id());
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- streaming::topics::helpers::get_topic_id(),
- );
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
// Route request to the correct shard
let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs
index c2935ef6a..abf12859e 100644
--- a/core/server/src/binary/handlers/streams/create_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs
@@ -21,15 +21,12 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
-
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker};
use crate::state::command::EntryCommand;
use crate::state::models::CreateStreamWithId;
use crate::streaming::session::Session;
@@ -74,20 +71,14 @@ impl ServerCommandHandler for CreateStream {
// Acquire stream lock to serialize filesystem operations
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard.create_stream(session, name).await?;
- let created_stream_id = stream.id();
+ let created_stream_id = shard.create_stream(session, name).await?;
let event = ShardEvent::CreatedStream {
id: created_stream_id,
- stream: stream.clone(),
};
shard.broadcast_event_to_all_shards(event).await?;
- let response = shard
- .streams
- .with_components_by_id(created_stream_id, |(root, stats)| {
- mapper::map_stream(&root, &stats)
- });
+ let response = shard.get_stream_from_shared_metadata(created_stream_id);
shard
.state
@@ -110,9 +101,8 @@ impl ServerCommandHandler for CreateStream {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateStreamResponse(stream) => {
- let created_stream_id = stream.id();
- let response = mapper::map_stream(stream.root(), stream.stats());
+ ShardResponse::CreateStreamResponse(created_stream_id) => {
+ let response = shard.get_stream_from_shared_metadata(created_stream_id);
shard
.state
diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index d1e23cb04..7723e07bc 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -27,7 +27,7 @@ use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
@@ -71,7 +71,7 @@ impl ServerCommandHandler for DeleteStream {
{
// Acquire stream lock to serialize filesystem operations
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard
+ let stream_info = shard
.delete_stream(session, &stream_id)
.await
.error(|e: &IggyError| {
@@ -79,12 +79,11 @@ impl ServerCommandHandler for DeleteStream {
})?;
info!(
"Deleted stream with name: {}, ID: {}",
- stream.root().name(),
- stream.id()
+ stream_info.name, stream_info.id
);
let event = ShardEvent::DeletedStream {
- id: stream.id(),
+ id: stream_info.id,
stream_id: stream_id.clone(),
};
shard.broadcast_event_to_all_shards(event).await?;
@@ -100,10 +99,10 @@ impl ServerCommandHandler for DeleteStream {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteStreamResponse(stream) => {
+ ShardResponse::DeleteStreamResponse(stream_id_num) => {
shard
.state
- .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream.id() as u32).try_into().unwrap() }))
+ .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream_id_num as u32).try_into().unwrap() }))
.await
.error(|e: &IggyError| {
format!("{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}, session: {session}")
diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs
index fe9cd1f83..17ca9910d 100644
--- a/core/server/src/binary/handlers/streams/get_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs
@@ -20,11 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::EntityComponentSystem;
use crate::streaming::session::Session;
-use crate::streaming::streams;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::IggyError;
@@ -47,18 +44,16 @@ impl ServerCommandHandler for GetStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let exists = shard.ensure_stream_exists(&self.stream_id).is_ok();
- if !exists {
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let Some(numeric_stream_id) = shard.shared_metadata.get_stream_id(&self.stream_id) else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
- }
- let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ };
+
let has_permission = shard
.permissioner
- .borrow()
- .get_stream(session.get_user_id(), stream_id)
+ .get_stream(session.get_user_id(), numeric_stream_id)
.error(|e: &IggyError| {
format!(
"permission denied to get stream with ID: {} for user with ID: {}, error: {e}",
@@ -71,9 +66,10 @@ impl ServerCommandHandler for GetStream {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let response = shard
- .streams
- .with_components_by_id(stream_id, |(root, stats)| mapper::map_stream(&root, &stats));
+
+ // Use SharedMetadata for stream info with stats (source of truth)
+ // This ensures stats are consistent across all shards
+ let response = shard.get_stream_from_shared_metadata(numeric_stream_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs
index f7bf06f2d..ddbe77359 100644
--- a/core/server/src/binary/handlers/streams/get_streams_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs
@@ -21,9 +21,7 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
use anyhow::Result;
use err_trail::ErrContext;
@@ -49,7 +47,6 @@ impl ServerCommandHandler for GetStreams {
shard.ensure_authenticated(session)?;
shard
.permissioner
- .borrow()
.get_streams(session.get_user_id())
.error(|e: &IggyError| {
format!(
@@ -58,10 +55,9 @@ impl ServerCommandHandler for GetStreams {
)
})?;
- let response = shard.streams.with_components(|stream_ref| {
- let (roots, stats) = stream_ref.into_components();
- mapper::map_streams(&roots, &stats)
- });
+ // Use SharedMetadata for stream info with stats (source of truth)
+ // This ensures stats are consistent across all shards
+ let response = shard.get_streams_from_shared_metadata();
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs
index 8e4de6574..7d298d532 100644
--- a/core/server/src/binary/handlers/streams/update_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs
@@ -24,12 +24,16 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_stream::UpdateStream;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -47,26 +51,69 @@ impl ServerCommandHandler for UpdateStream {
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
- let stream_id = self.stream_id.clone();
- shard
- .update_stream(session, &self.stream_id, self.name.clone())
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to update stream with id: {stream_id}, session: {session}")
- })?;
- let event = ShardEvent::UpdatedStream {
- stream_id: self.stream_id.clone(),
- name: self.name.clone(),
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::UpdateStream {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ name: self.name.clone(),
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
- .await
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}")
- })?;
- sender.send_empty_ok_response().await?;
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::UpdateStream {
+ stream_id, name, ..
+ } = payload
+ {
+ shard.update_stream(session, &stream_id, name.clone())?;
+
+ let event = ShardEvent::UpdatedStream {
+ stream_id: stream_id.clone(),
+ name: name.clone(),
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}"
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected an UpdateStream request inside of UpdateStream handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::UpdateStreamResponse => {
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected an UpdateStreamResponse inside of UpdateStream handler, impossible state"
+ ),
+ },
+ }
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index fa5c0270a..533349552 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -21,19 +21,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
-
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
-use crate::streaming::streams;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_topic::CreateTopic;
@@ -90,7 +87,7 @@ impl ServerCommandHandler for CreateTopic {
// Acquire topic lock to serialize filesystem operations
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard
+ let topic_id = shard
.create_topic(
session,
&stream_id,
@@ -101,21 +98,26 @@ impl ServerCommandHandler for CreateTopic {
replication_factor,
)
.await?;
- self.message_expiry = topic.root().message_expiry();
- self.max_topic_size = topic.root().max_topic_size();
let stream_id_num = shard
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
- let topic_id = topic.id();
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+
+ // Get topic metadata to update command fields
+ let metadata = shard.shared_metadata.load();
+ if let Some(topic_meta) = metadata.topics.get(&(stream_id_num, topic_id)) {
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
+ }
let event = ShardEvent::CreatedTopic {
stream_id: stream_id.clone(),
- topic,
+ topic_id,
};
shard.broadcast_event_to_all_shards(event).await?;
- let partitions = shard
+ let partition_infos = shard
.create_partitions(
session,
&stream_id,
@@ -126,15 +128,11 @@ impl ServerCommandHandler for CreateTopic {
let event = ShardEvent::CreatedPartitions {
stream_id: stream_id.clone(),
topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions,
+ partitions: partition_infos.clone(),
};
shard.broadcast_event_to_all_shards(event).await?;
- let response = shard.streams.with_topic_by_id(
- &stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- |(root, _, stats)| mapper::map_topic(&root, &stats),
- );
+ let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id);
shard
.state
.apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId {
@@ -155,20 +153,22 @@ impl ServerCommandHandler for CreateTopic {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateTopicResponse(topic) => {
- let topic_id = topic.id();
- self.message_expiry = topic.root().message_expiry();
- self.max_topic_size = topic.root().max_topic_size();
-
- let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
-
- let response = shard.streams.with_topic_by_id(
- &self.stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- |(root, _, stats)| mapper::map_topic(&root, &stats),
- );
+ ShardResponse::CreateTopicResponse(topic_id) => {
+ // Use SharedMetadata for ID resolution (slabs are empty on non-primary shards)
+ let stream_id_num = shard
+ .shared_metadata
+ .get_stream_id(&self.stream_id)
+ .expect("Stream must exist after CreateTopic");
+
+ // Get topic metadata to update command fields
+ let metadata = shard.shared_metadata.load();
+ if let Some(topic_meta) = metadata.topics.get(&(stream_id_num, topic_id)) {
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
+ }
+
+ // Use SharedMetadata for topic response (slabs are empty on non-primary shards)
+ let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id);
shard
.state
@@ -179,7 +179,7 @@ impl ServerCommandHandler for CreateTopic {
.await
.error(|e: &IggyError| {
format!(
- "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id}, topic_id: {topic_id:?}"
+ "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id_num}, topic_id: {topic_id:?}"
)
})?;
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index 668dbd707..ea1a98a45 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -28,10 +28,9 @@ use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::streams;
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_topic::DeleteTopic;
@@ -79,16 +78,12 @@ impl ServerCommandHandler for DeleteTopic {
// Acquire topic lock to serialize filesystem operations
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard.delete_topic(session, &stream_id, &topic_id).await?;
- let stream_id_num = shard
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
- let topic_id_num = topic.root().id();
+ let topic_info = shard.delete_topic(session, &stream_id, &topic_id).await?;
+ let topic_id_num = topic_info.id;
+ let stream_id_num = topic_info.stream_id;
info!(
"Deleted topic with name: {}, ID: {} in stream with ID: {}",
- topic.root().name(),
- topic_id_num,
- stream_id_num
+ topic_info.name, topic_id_num, stream_id_num
);
let event = ShardEvent::DeletedTopic {
@@ -113,14 +108,13 @@ impl ServerCommandHandler for DeleteTopic {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteTopicResponse(topic) => {
+ ShardResponse::DeleteTopicResponse(topic_id_num) => {
shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
.await
.error(|e: &IggyError| format!(
- "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {}, session: {session}",
- topic.id()
+ "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {topic_id_num}, session: {session}"
))?;
sender.send_empty_ok_response().await?;
diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs
index fcc42ac0b..c02f9c158 100644
--- a/core/server/src/binary/handlers/topics/get_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs
@@ -20,10 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::streams;
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -45,41 +43,33 @@ impl ServerCommandHandler for GetTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let exists = shard
- .ensure_topic_exists(&self.stream_id, &self.topic_id)
- .is_ok();
- if !exists {
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let Some(numeric_stream_id) = shard.shared_metadata.get_stream_id(&self.stream_id) else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
- }
+ };
+
+ let Some(numeric_topic_id) = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &self.topic_id)
+ else {
+ sender.send_empty_ok_response().await?;
+ return Ok(HandlerResult::Finished);
+ };
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
let has_permission = shard
.permissioner
- .borrow()
- .get_topic(
- session.get_user_id(),
- numeric_stream_id,
- self.topic_id
- .get_u32_value()
- .unwrap_or(0)
- .try_into()
- .unwrap(),
- )
+ .get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.is_ok();
if !has_permission {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let response =
- shard
- .streams
- .with_topic_by_id(&self.stream_id, &self.topic_id, |(root, _, stats)| {
- mapper::map_topic(&root, &stats)
- });
+ // Use SharedMetadata for topic info with stats (source of truth)
+ // This ensures stats are consistent across all shards
+ let response = shard.get_topic_from_shared_metadata(numeric_stream_id, numeric_topic_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs
index 96c3f9325..71939bc72 100644
--- a/core/server/src/binary/handlers/topics/get_topics_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs
@@ -20,11 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
-use crate::streaming::streams;
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -47,20 +44,20 @@ impl ServerCommandHandler for GetTopics {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
shard.ensure_stream_exists(&self.stream_id)?;
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let Some(numeric_stream_id) = shard.shared_metadata.get_stream_id(&self.stream_id) else {
+ sender.send_empty_ok_response().await?;
+ return Ok(HandlerResult::Finished);
+ };
+
shard
.permissioner
- .borrow()
.get_topics(session.get_user_id(), numeric_stream_id)?;
- let response = shard.streams.with_topics(&self.stream_id, |topics| {
- topics.with_components(|topics| {
- let (roots, _, stats) = topics.into_components();
- mapper::map_topics(&roots, &stats)
- })
- });
+ // Use SharedMetadata for topic info with stats (source of truth)
+ // This ensures stats are consistent across all shards
+ let response = shard.get_topics_from_shared_metadata(numeric_stream_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs
index 90f5e11ec..92023bd82 100644
--- a/core/server/src/binary/handlers/topics/update_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs
@@ -30,7 +30,6 @@ use crate::shard::transmission::message::{
};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_topic::UpdateTopic;
@@ -84,6 +83,16 @@ impl ServerCommandHandler for UpdateTopic {
..
} = payload
{
+ // Get numeric IDs BEFORE update (name might change during update)
+ let stream_id_num = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let topic_id_num = shard
+ .shared_metadata
+ .get_topic_id(stream_id_num, &topic_id)
+ .expect("Topic must exist");
+
shard.update_topic(
session,
&stream_id,
@@ -95,27 +104,14 @@ impl ServerCommandHandler for UpdateTopic {
replication_factor,
)?;
- let name_changed = !name.is_empty();
- let lookup_topic_id = if name_changed {
- Identifier::named(&name).unwrap()
- } else {
- topic_id.clone()
- };
-
- self.message_expiry = shard.streams.with_topic_by_id(
- &stream_id,
- &lookup_topic_id,
- topics::helpers::get_message_expiry(),
- );
- self.max_topic_size = shard.streams.with_topic_by_id(
- &stream_id,
- &lookup_topic_id,
- topics::helpers::get_max_topic_size(),
- );
-
- let stream_id_num = shard
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
+ let metadata = shard.shared_metadata.load();
+ let topic_key = (stream_id_num, topic_id_num);
+ let topic_meta = metadata
+ .topics
+ .get(&topic_key)
+ .expect("Topic must exist after update");
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
let event = ShardEvent::UpdatedTopic {
stream_id: stream_id.clone(),
@@ -144,9 +140,11 @@ impl ServerCommandHandler for UpdateTopic {
}
ShardSendRequestResult::Response(response) => match response {
ShardResponse::UpdateTopicResponse => {
+ // Use SharedMetadata on non-shard-0 to avoid slab access
let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ .shared_metadata
+ .get_stream_id(&self.stream_id)
+ .unwrap_or_default();
shard
.state
diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs
index 0dadd98bb..1231bfc9a 100644
--- a/core/server/src/binary/handlers/users/change_password_handler.rs
+++ b/core/server/src/binary/handlers/users/change_password_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use crate::streaming::utils::crypto;
@@ -67,13 +66,6 @@ impl ServerCommandHandler for ChangePassword {
info!("Changed password for user with ID: {}.", self.user_id);
- let event = ShardEvent::ChangedPassword {
- user_id: self.user_id.clone(),
- current_password: self.current_password.clone(),
- new_password: self.new_password.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
// For the security of the system, we hash the password before storing it in metadata.
shard
.state
diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs
index 3cace71b3..6ff15da51 100644
--- a/core/server/src/binary/handlers/users/create_user_handler.rs
+++ b/core/server/src/binary/handlers/users/create_user_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
@@ -91,16 +90,6 @@ impl ServerCommandHandler for CreateUser {
})?;
let user_id = user.id;
-
- let event = ShardEvent::CreatedUser {
- user_id,
- username: username.clone(),
- password: password.clone(),
- status,
- permissions: permissions.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
let response = mapper::map_user(&user);
shard
diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs
index a7fa3ec1b..9ba1db0b8 100644
--- a/core/server/src/binary/handlers/users/delete_user_handler.rs
+++ b/core/server/src/binary/handlers/users/delete_user_handler.rs
@@ -25,7 +25,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
@@ -81,8 +80,6 @@ impl ServerCommandHandler for DeleteUser {
})?;
info!("Deleted user: {} with ID: {}.", user.username, user.id);
- let event = ShardEvent::DeletedUser { user_id };
- shard.broadcast_event_to_all_shards(event).await?;
shard
.state
diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs
index 30829a840..1c5097dd6 100644
--- a/core/server/src/binary/handlers/users/update_permissions_handler.rs
+++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs
@@ -25,7 +25,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
@@ -56,11 +55,6 @@ impl ServerCommandHandler for UpdatePermissions {
self.user_id
))?;
info!("Updated permissions for user with ID: {}.", self.user_id);
- let event = ShardEvent::UpdatedPermissions {
- user_id: self.user_id.clone(),
- permissions: self.permissions.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
shard
.state
diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs
index bdb929136..45f0c6b96 100644
--- a/core/server/src/binary/handlers/users/update_user_handler.rs
+++ b/core/server/src/binary/handlers/users/update_user_handler.rs
@@ -25,7 +25,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
@@ -66,13 +65,6 @@ impl ServerCommandHandler for UpdateUser {
info!("Updated user: {} with ID: {}.", user.username, user.id);
- let event = ShardEvent::UpdatedUser {
- user_id: self.user_id.clone(),
- username: self.username.clone(),
- status: self.status,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
let user_id = self.user_id.clone();
shard
.state
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 7b17c038e..a72627661 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -16,22 +16,12 @@
* under the License.
*/
-use std::sync::{Arc, atomic::AtomicU64};
-
-use crate::slab::Keyed;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
+use crate::metadata::ConsumerGroupMeta;
use crate::streaming::clients::client_manager::Client;
-use crate::streaming::partitions::partition::PartitionRoot;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
-use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
-use crate::streaming::streams::stream;
-use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot, Member};
-use crate::streaming::topics::topic::{self, TopicRoot};
use crate::streaming::users::user::User;
-use arcshift::SharedGetGuard;
use bytes::{BufMut, Bytes, BytesMut};
use iggy_common::{BytesSerializable, ConsumerOffsetInfo, Stats, TransportProtocol, UserId};
-use slab::Slab;
pub fn map_stats(stats: &Stats) -> Bytes {
let mut bytes = BytesMut::with_capacity(104);
@@ -152,145 +142,29 @@ pub fn map_personal_access_tokens(personal_access_tokens: Vec<PersonalAccessToke
bytes.freeze()
}
-pub fn map_streams(roots: &Slab<stream::StreamRoot>, stats: &Slab<Arc<StreamStats>>) -> Bytes {
+/// Map consumer group from SharedMetadata format.
+pub fn map_consumer_group_from_meta(meta: &ConsumerGroupMeta) -> Bytes {
let mut bytes = BytesMut::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_stream(root, stat, &mut bytes);
- }
- bytes.freeze()
-}
-pub fn map_stream(root: &stream::StreamRoot, stats: &StreamStats) -> Bytes {
- let mut bytes = BytesMut::new();
- extend_stream(root, stats, &mut bytes);
- root.topics().with_components(|topics| {
- let (roots, _, stats, ..) = topics.into_components();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_topic(root, stat, &mut bytes);
- }
- });
- bytes.freeze()
-}
+ // Header: id, partitions_count, members_count, name_len, name
+ bytes.put_u32_le(meta.id as u32);
+ bytes.put_u32_le(meta.partitions.len() as u32);
+ bytes.put_u32_le(meta.members.len() as u32);
+ bytes.put_u8(meta.name.len() as u8);
+ bytes.put_slice(meta.name.as_bytes());
-pub fn map_topics(roots: &Slab<TopicRoot>, stats: &Slab<Arc<TopicStats>>) -> Bytes {
- let mut bytes = BytesMut::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_topic(root, stat, &mut bytes);
- }
- bytes.freeze()
-}
-
-pub fn map_topic(root: &topic::TopicRoot, stats: &TopicStats) -> Bytes {
- let mut bytes = BytesMut::new();
- extend_topic(root, stats, &mut bytes);
- root.partitions().with_components(|partitions| {
- let (roots, stats, _, offsets, _, _, _) = partitions.into_components();
- for (root, stat, offset) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- .zip(offsets.iter().map(|(_, val)| val))
- .map(|((root, stat), offset)| (root, stat, offset))
- {
- extend_partition(root, stat, offset, &mut bytes);
- }
- });
-
- bytes.freeze()
-}
-
-pub fn map_consumer_group(root: &ConsumerGroupRoot, members: &ConsumerGroupMembers) -> Bytes {
- let mut bytes = BytesMut::new();
- let members = members.inner().shared_get();
- extend_consumer_group(root, &members, &mut bytes);
-
- for (_, member) in members.iter() {
+ // Members
+ for member in meta.members.iter() {
bytes.put_u32_le(member.id as u32);
bytes.put_u32_le(member.partitions.len() as u32);
- for partition in &member.partitions {
- bytes.put_u32_le(*partition as u32);
+ for &partition_id in &member.partitions {
+ bytes.put_u32_le(partition_id as u32);
}
}
- bytes.freeze()
-}
-pub fn map_consumer_groups(
- roots: &Slab<ConsumerGroupRoot>,
- members: &Slab<ConsumerGroupMembers>,
-) -> Bytes {
- let mut bytes = BytesMut::new();
- for (root, member) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(members.iter().map(|(_, val)| val.inner().shared_get()))
- {
- extend_consumer_group(root, &member, &mut bytes);
- }
bytes.freeze()
}
-fn extend_stream(root: &stream::StreamRoot, stats: &StreamStats, bytes: &mut BytesMut) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(root.topics_count() as u32);
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
- bytes.put_u8(root.name().len() as u8);
- bytes.put_slice(root.name().as_bytes());
-}
-
-fn extend_topic(root: &TopicRoot, stats: &TopicStats, bytes: &mut BytesMut) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(root.partitions().len() as u32);
- bytes.put_u64_le(root.message_expiry().into());
- bytes.put_u8(root.compression_algorithm().as_code());
- bytes.put_u64_le(root.max_topic_size().into());
- bytes.put_u8(root.replication_factor());
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
- bytes.put_u8(root.name().len() as u8);
- bytes.put_slice(root.name().as_bytes());
-}
-
-fn extend_partition(
- root: &PartitionRoot,
- stats: &PartitionStats,
- offset: &Arc<AtomicU64>,
- bytes: &mut BytesMut,
-) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(stats.segments_count_inconsistent());
- bytes.put_u64_le(offset.load(std::sync::atomic::Ordering::Relaxed));
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
-}
-
-fn extend_consumer_group(
- root: &ConsumerGroupRoot,
- members: &SharedGetGuard<'_, Slab<Member>>,
- bytes: &mut BytesMut,
-) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u32_le(root.partitions().len() as u32);
- bytes.put_u32_le(members.len() as u32);
- bytes.put_u8(root.key().len() as u8);
- bytes.put_slice(root.key().as_bytes());
-}
-
fn extend_client(client: &Client, bytes: &mut BytesMut) {
bytes.put_u32_le(client.session.client_id);
bytes.put_u32_le(client.user_id.unwrap_or(u32::MAX));
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 67c832231..ee750f346 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -27,45 +27,28 @@ use crate::{
system::{INDEX_EXTENSION, LOG_EXTENSION, SystemConfig},
},
io::fs_utils::{self, DirEntry},
+ metadata::{ConsumerGroupMeta, SharedMetadata, UserMeta},
server_error::ServerError,
shard::{
+ namespace::IggyNamespace,
system::info::SystemInfo,
transmission::{
connector::{ShardConnector, StopSender},
frame::ShardFrame,
},
},
- slab::{
- streams::Streams,
- traits_ext::{
- EntityComponentSystem, EntityComponentSystemMutCell, Insert, InsertCell, IntoComponents,
- },
- users::Users,
- },
state::system::{StreamState, TopicState, UserState},
streaming::{
- partitions::{
- consumer_offset::ConsumerOffset,
- helpers::create_message_deduplicator,
- journal::MemoryMessageJournal,
- log::SegmentedLog,
- partition,
- storage::{load_consumer_group_offsets, load_consumer_offsets},
- },
+ partitions::{journal::MemoryMessageJournal, log::SegmentedLog},
persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
- personal_access_tokens::personal_access_token::PersonalAccessToken,
- polling_consumer::ConsumerGroupId,
segments::{Segment, storage::Storage},
- stats::{PartitionStats, StreamStats, TopicStats},
+ stats::PartitionStats,
storage::SystemStorage,
- streams::stream,
- topics::{consumer_group, topic},
users::user::User,
utils::{crypto, file::overwrite},
},
versioning::SemanticVersion,
};
-use ahash::HashMap;
use compio::{fs::create_dir_all, runtime::Runtime};
use err_trail::ErrContext;
use iggy_common::{
@@ -78,178 +61,6 @@ use iggy_common::{
use std::{env, path::Path, sync::Arc};
use tracing::{info, warn};
-pub async fn load_streams(
- state: impl IntoIterator<Item = StreamState>,
- config: &SystemConfig,
-) -> Result<Streams, IggyError> {
- let streams = Streams::default();
- for StreamState {
- name,
- created_at,
- id,
- topics,
- } in state
- {
- info!(
- "Loading stream with ID: {}, name: {} from state...",
- id, name
- );
- let stream_id = id;
- let stats = Arc::new(StreamStats::default());
- let stream = stream::Stream::new(name.clone(), stats.clone(), created_at);
- let new_id = streams.insert(stream);
- assert_eq!(
- new_id, stream_id as usize,
- "load_streams: id mismatch when inserting stream, mismatch for stream with ID: {}, name: {}",
- stream_id, name
- );
- info!(
- "Loaded stream with ID: {}, name: {} from state...",
- id, name
- );
-
- let topics = topics.into_values();
- for TopicState {
- id,
- name,
- created_at,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor,
- consumer_groups,
- partitions,
- } in topics
- {
- info!(
- "Loading topic with ID: {}, name: {} from state...",
- id, name
- );
- let topic_id = id;
- let parent_stats = stats.clone();
- let stats = Arc::new(TopicStats::new(parent_stats));
- let topic_id = streams.with_components_by_id_mut(stream_id as usize, |(mut root, ..)| {
- let topic = topic::Topic::new(
- name.clone(),
- stats.clone(),
- created_at,
- replication_factor.unwrap_or(1),
- message_expiry,
- compression_algorithm,
- max_topic_size,
- );
- let new_id = root.topics_mut().insert(topic);
- assert_eq!(
- new_id, topic_id as usize,
- "load_streams: topic id mismatch when inserting topic, mismatch for topic with ID: {}, name: {}",
- topic_id, &name
- );
- new_id
- });
- info!("Loaded topic with ID: {}, name: {} from state...", id, name);
-
- let parent_stats = stats.clone();
- let cgs = consumer_groups.into_values();
- let partitions = partitions.into_values();
-
- // Load each partition asynchronously and insert immediately
- for partition_state in partitions {
- info!(
- "Loading partition with ID: {}, for topic with ID: {} from state...",
- partition_state.id, topic_id
- );
-
- let partition_id = partition_state.id;
- let partition = load_partition(
- config,
- stream_id as usize,
- topic_id,
- partition_state,
- parent_stats.clone(),
- )
- .await?;
- streams.with_components_by_id(stream_id as usize, |(root, ..)| {
- root.topics()
- .with_components_by_id_mut(topic_id, |(mut root, ..)| {
- let new_id = root.partitions_mut().insert(partition);
- assert_eq!(
- new_id, partition_id as usize,
- "load_streams: partition id mismatch when inserting partition, mismatch for partition with ID: {}, for topic with ID: {}, for stream with ID: {}",
- partition_id, topic_id, stream_id
- );
- });
- });
-
- info!(
- "Loaded partition with ID: {}, for topic with ID: {} from state...",
- partition_id, topic_id
- );
- }
- let partition_ids = streams.with_components_by_id(stream_id as usize, |(root, ..)| {
- root.topics().with_components_by_id(topic_id, |(root, ..)| {
- root.partitions().with_components(|components| {
- let (root, ..) = components.into_components();
- root.iter().map(|(_, root)| root.id()).collect::<Vec<_>>()
- })
- })
- });
-
- for cg_state in cgs {
- info!(
- "Loading consumer group with ID: {}, name: {} for topic with ID: {} from state...",
- cg_state.id, cg_state.name, topic_id
- );
- streams.with_components_by_id(stream_id as usize, |(root, ..)| {
- root.topics()
- .with_components_by_id_mut(topic_id, |(mut root, ..)| {
- let id = cg_state.id;
- let cg = consumer_group::ConsumerGroup::new(cg_state.name.clone(), Default::default(), partition_ids.clone());
- let new_id = root.consumer_groups_mut().insert(cg);
- assert_eq!(
- new_id, id as usize,
- "load_streams: consumer group id mismatch when inserting consumer group, mismatch for consumer group with ID: {}, name: {} for topic with ID: {}, for stream with ID: {}",
- id, cg_state.name, topic_id, stream_id
- );
- });
- });
- info!(
- "Loaded consumer group with ID: {}, name: {} for topic with ID: {} from state...",
- cg_state.id, cg_state.name, topic_id
- );
- }
- }
- }
- Ok(streams)
-}
-
-pub fn load_users(state: impl IntoIterator<Item = UserState>) -> Users {
- let users = Users::new();
- for user_state in state {
- let UserState {
- id,
- username,
- password_hash,
- status,
- created_at,
- permissions,
- personal_access_tokens,
- } = user_state;
- let mut user = User::with_password(id, &username, password_hash, status, permissions);
- user.created_at = created_at;
- user.personal_access_tokens = personal_access_tokens
- .into_values()
- .map(|token| {
- (
- Arc::new(token.token_hash.clone()),
- PersonalAccessToken::raw(id, &token.name, &token.token_hash, token.expiry_at),
- )
- })
- .collect();
- users.insert(user);
- }
- users
-}
-
pub fn create_shard_connections(
shard_assignment: &[ShardInfo],
) -> (Vec<ShardConnector<ShardFrame>>, Vec<(u16, StopSender)>) {
@@ -648,86 +459,128 @@ pub async fn load_segments(
Ok(log)
}
-async fn load_partition(
- config: &SystemConfig,
- stream_id: usize,
- topic_id: usize,
- partition_state: crate::state::system::PartitionState,
- parent_stats: Arc<TopicStats>,
-) -> Result<partition::Partition, IggyError> {
- let stats = Arc::new(PartitionStats::new(parent_stats));
- let partition_id = partition_state.id;
-
- let partition_path = config.get_partition_path(stream_id, topic_id, partition_id as usize);
- let log_files = collect_log_files(&partition_path).await?;
- let should_increment_offset = !log_files.is_empty()
- && log_files
- .first()
- .map(|entry| {
- let log_file_name = entry
- .path
- .file_stem()
- .unwrap()
- .to_string_lossy()
- .to_string();
-
- let start_offset = log_file_name.parse::<u64>().unwrap();
-
- let messages_file_path = config.get_messages_file_path(
- stream_id,
- topic_id,
- partition_id as usize,
- start_offset,
- );
- let metadata = std::fs::metadata(&messages_file_path)
- .expect("failed to get metadata for first segment in log");
- metadata.len() > 0
- })
- .unwrap_or_else(|| false);
+/// Loads users directly into SharedMetadata without using slabs.
+pub fn load_users_to_metadata(
+ state: impl IntoIterator<Item = UserState>,
+ shared_metadata: &SharedMetadata,
+) {
+ for user_state in state {
+ let UserState {
+ id,
+ username,
+ password_hash,
+ status,
+ created_at,
+ permissions,
+ personal_access_tokens: _, // Personal access tokens are handled separately
+ } = user_state;
- info!(
- "Loading partition with ID: {} for stream with ID: {} and topic with ID: {}, for path: {} from disk...",
- partition_id, stream_id, topic_id, partition_path
- );
+ let user_meta = UserMeta {
+ id,
+ username: Arc::from(username.as_str()),
+ password_hash: Arc::from(password_hash.as_str()),
+ status,
+ permissions: permissions.map(Arc::new),
+ created_at,
+ };
+ shared_metadata.add_user(id, user_meta);
+ }
- // Load consumer offsets
- let message_deduplicator = create_message_deduplicator(config);
- let consumer_offset_path =
- config.get_consumer_offsets_path(stream_id, topic_id, partition_id as usize);
- let consumer_group_offsets_path =
- config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id as usize);
-
- let consumer_offset = Arc::new(
- load_consumer_offsets(&consumer_offset_path)?
- .into_iter()
- .map(|offset| (offset.consumer_id as usize, offset))
- .collect::<HashMap<usize, ConsumerOffset>>()
- .into(),
- );
+ let metadata = shared_metadata.load();
+ info!("Loaded into SharedMetadata: {} users", metadata.users.len());
+}
- let consumer_group_offset = Arc::new(
- load_consumer_group_offsets(&consumer_group_offsets_path)?
- .into_iter()
- .collect::<HashMap<ConsumerGroupId, ConsumerOffset>>()
- .into(),
- );
+/// Loads stream/topic/partition/consumer_group metadata into SharedMetadata.
+/// Does NOT load partition data (logs, segments) - that's done per-shard via load_segments().
+pub fn load_metadata_only(
+ state: impl IntoIterator<Item = StreamState>,
+ shared_metadata: &SharedMetadata,
+) {
+ for StreamState {
+ name,
+ created_at,
+ id,
+ topics,
+ } in state
+ {
+ info!("Loading stream with ID: {}, name: {} metadata...", id, name);
+ let stream_id = id as usize;
- let log = Default::default();
- let partition = partition::Partition::new(
- partition_state.created_at,
- should_increment_offset,
- stats,
- message_deduplicator,
- Arc::new(Default::default()),
- consumer_offset,
- consumer_group_offset,
- log,
- );
+ // Register stream (creates stats + meta atomically)
+ let _stream_stats =
+ shared_metadata.register_stream(stream_id, Arc::from(name.as_str()), created_at);
+
+ for TopicState {
+ id,
+ name,
+ created_at,
+ compression_algorithm,
+ message_expiry,
+ max_topic_size,
+ replication_factor,
+ consumer_groups,
+ partitions,
+ } in topics.into_values()
+ {
+ info!("Loading topic with ID: {}, name: {} metadata...", id, name);
+ let topic_id = id as usize;
+ let partitions_count = partitions.len() as u32;
+
+ // Register topic (creates stats + meta atomically)
+ let _topic_stats = shared_metadata.register_topic(
+ stream_id,
+ topic_id,
+ Arc::from(name.as_str()),
+ created_at,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor.unwrap_or(1),
+ partitions_count,
+ );
+
+ // Register partitions (creates stats + meta atomically)
+ // Note: Partition data (logs, offsets) loaded per-shard via load_segments()
+ let mut partition_ids = Vec::new();
+ for partition_state in partitions.into_values() {
+ let partition_id = partition_state.id as usize;
+ partition_ids.push(partition_id);
+
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ let _partition_stats =
+ shared_metadata.register_partition(ns, partition_state.created_at);
+ }
+
+ // Sort partition IDs for consumer group round-robin
+ partition_ids.sort_unstable();
+
+ // Register consumer groups
+ for cg_state in consumer_groups.into_values() {
+ info!(
+ "Loading consumer group with ID: {}, name: {} for topic with ID: {} metadata...",
+ cg_state.id, cg_state.name, topic_id
+ );
+ let group_id = cg_state.id as usize;
+ let cg_meta = ConsumerGroupMeta {
+ id: group_id,
+ stream_id,
+ topic_id,
+ name: Arc::from(cg_state.name.as_str()),
+ partitions: partition_ids.clone(),
+ members: imbl::Vector::new(),
+ };
+ shared_metadata.add_consumer_group((stream_id, topic_id, group_id), cg_meta);
+ }
+ }
+ }
+ let metadata = shared_metadata.load();
info!(
- "Loaded partition with ID: {} for stream with ID: {} and topic with ID: {}",
- partition_id, stream_id, topic_id
+ "Loaded metadata: {} streams, {} topics, {} partitions, {} consumer groups (version: {})",
+ metadata.streams.len(),
+ metadata.topics.len(),
+ metadata.partitions.len(),
+ metadata.consumer_groups.len(),
+ metadata.version
);
-
- Ok(partition)
}
diff --git a/core/server/src/configs/system.rs b/core/server/src/configs/system.rs
index c1b089a79..fd5b416b8 100644
--- a/core/server/src/configs/system.rs
+++ b/core/server/src/configs/system.rs
@@ -17,9 +17,6 @@
*/
use crate::configs::server::MemoryPoolConfig;
-use crate::slab::partitions;
-use crate::slab::streams;
-use crate::slab::topics;
use super::cache_indexes::CacheIndexesConfig;
use super::sharding::ShardingConfig;
@@ -218,9 +215,9 @@ impl SystemConfig {
pub fn get_partition_path(
&self,
- stream_id: streams::ContainerId,
- topic_id: topics::ContainerId,
- partition_id: partitions::ContainerId,
+ stream_id: usize,
+ topic_id: usize,
+ partition_id: usize,
) -> String {
format!(
"{}/{}",
diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs
index 5faf48adb..e744f4ca7 100644
--- a/core/server/src/http/consumer_groups.rs
+++ b/core/server/src/http/consumer_groups.rs
@@ -21,7 +21,6 @@ use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::mapper;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents};
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::polling_consumer::ConsumerGroupId;
@@ -64,45 +63,42 @@ async fn get_consumer_group(
let identifier_group_id = Identifier::from_str_value(&group_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
-
- // Check permissions and existence
- state.shard.shard().ensure_authenticated(&session)?;
- let exists = state
- .shard
- .shard()
- .ensure_consumer_group_exists(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id,
- )
- .is_ok();
- if !exists {
- return Err(CustomError::ResourceNotFound);
- }
-
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
-
- state
- .shard
- .shard()
- .permissioner
- .borrow()
- .get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)?;
-
- let consumer_group = state.shard.shard().streams.with_consumer_group_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ let shard = state.shard.shard();
+
+ // Check authentication
+ shard.ensure_authenticated(&session)?;
+
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&identifier_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &identifier_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_group_id = shard
+ .shared_metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &identifier_group_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ // Check permissions
+ shard.permissioner.get_consumer_group(
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id,
+ )?;
+
+ // Get consumer group from SharedMetadata
+ let metadata = shard.shared_metadata.load();
+ let cg_meta = metadata
+ .consumer_groups
+ .get(&(numeric_stream_id, numeric_topic_id, numeric_group_id))
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let consumer_group = mapper::map_consumer_group_details_from_metadata(cg_meta);
Ok(Json(consumer_group))
}
@@ -116,41 +112,33 @@ async fn get_consumer_groups(
let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
-
- // Check permissions and existence
- state.shard.shard().ensure_authenticated(&session)?;
- state
- .shard
- .shard()
- .ensure_topic_exists(&identifier_stream_id, &identifier_topic_id)?;
-
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
-
- state
- .shard
- .shard()
- .permissioner
- .borrow()
- .get_consumer_groups(session.get_user_id(), numeric_stream_id, numeric_topic_id)?;
-
- let consumer_groups = state.shard.shard().streams.with_consumer_groups(
- &identifier_stream_id,
- &identifier_topic_id,
- |cgs| {
- cgs.with_components(|cgs| {
- let (roots, members) = cgs.into_components();
- mapper::map_consumer_groups(roots, members)
- })
- },
- );
+ let shard = state.shard.shard();
+
+ // Check authentication
+ shard.ensure_authenticated(&session)?;
+
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&identifier_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &identifier_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ // Check permissions
+ shard.permissioner.get_consumer_groups(
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id,
+ )?;
+
+ // Get consumer groups from SharedMetadata
+ let metadata = shard.shared_metadata.load();
+ let consumer_groups =
+ mapper::map_consumer_groups_from_metadata(numeric_stream_id, numeric_topic_id, &metadata);
Ok(Json(consumer_groups))
}
@@ -170,7 +158,7 @@ async fn create_consumer_group(
let session = Session::stateless(identity.user_id, identity.ip_address);
// Create consumer group using the new API
- let consumer_group = state.shard.shard().create_consumer_group(
+ let group_id = state.shard.shard().create_consumer_group(
&session,
&command.stream_id,
&command.topic_id,
@@ -178,8 +166,6 @@ async fn create_consumer_group(
)
.error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to create consumer group, stream ID: {}, topic ID: {}, name: {}", stream_id, topic_id, command.name))?;
- let group_id = consumer_group.id();
-
// Send event for consumer group creation
{
let broadcast_future = SendWrapper::new(async {
@@ -187,7 +173,7 @@ async fn create_consumer_group(
let event = ShardEvent::CreatedConsumerGroup {
stream_id: command.stream_id.clone(),
topic_id: command.topic_id.clone(),
- cg: consumer_group.clone(),
+ cg_id: group_id,
};
let _responses = state
.shard
@@ -198,14 +184,22 @@ async fn create_consumer_group(
broadcast_future.await;
}
- // Get the created consumer group details
- let group_id_identifier = Identifier::numeric(group_id as u32).unwrap();
- let consumer_group_details = state.shard.shard().streams.with_consumer_group_by_id(
- &command.stream_id,
- &command.topic_id,
- &group_id_identifier,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ // Get the created consumer group details from SharedMetadata
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &command.topic_id)
+ .expect("Topic must exist");
+ let metadata = shard.shared_metadata.load();
+ let cg_meta = metadata
+ .consumer_groups
+ .get(&(numeric_stream_id, numeric_topic_id, group_id))
+ .expect("Consumer group must exist after creation");
+ let consumer_group_details = mapper::map_consumer_group_details_from_metadata(cg_meta);
// Apply state change
let entry_command = EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
@@ -240,7 +234,7 @@ async fn delete_consumer_group(
let session = Session::stateless(identity.user_id, identity.ip_address);
// Delete using the new API
- let consumer_group = state.shard.shard().delete_consumer_group(
+ let cg_meta = state.shard.shard().delete_consumer_group(
&session,
&identifier_stream_id,
&identifier_topic_id,
@@ -248,27 +242,14 @@ async fn delete_consumer_group(
)
.error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {group_id} for topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
- let cg_id = consumer_group.id();
-
- // Remove all consumer group members from ClientManager
- let stream_id_usize = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
- let topic_id_usize = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
+ let cg_id = cg_meta.id;
- // TODO: Tech debt, repeated code from `delete_consumer_group_handler.rs`
- // Get members from the deleted consumer group and make them leave
- let slab = consumer_group.members().inner().shared_get();
- for (_, member) in slab.iter() {
+ // Make members leave the consumer group
+ for member in cg_meta.members.iter() {
if let Err(err) = state.shard.shard().client_manager.leave_consumer_group(
member.client_id,
- stream_id_usize,
- topic_id_usize,
+ cg_meta.stream_id,
+ cg_meta.topic_id,
cg_id,
) {
tracing::warn!(
@@ -281,12 +262,11 @@ async fn delete_consumer_group(
let cg_id_spez = ConsumerGroupId(cg_id);
// Clean up consumer group offsets from all partitions using the specialized method
- let partition_ids = consumer_group.partitions();
state.shard.shard().delete_consumer_group_offsets(
cg_id_spez,
&identifier_stream_id,
&identifier_topic_id,
- partition_ids,
+ &cg_meta.partitions,
).await.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs
index e7656a554..dff4150fd 100644
--- a/core/server/src/http/consumer_offsets.rs
+++ b/core/server/src/http/consumer_offsets.rs
@@ -118,6 +118,8 @@ async fn delete_consumer_offset(
Path((stream_id, topic_id, consumer_id)): Path<(String, String, String)>,
query: Query<DeleteConsumerOffset>,
) -> Result<StatusCode, CustomError> {
+ let stream_id_ident = Identifier::from_str_value(&stream_id)?;
+ let topic_id_ident = Identifier::from_str_value(&topic_id)?;
let consumer = Consumer::new(consumer_id.try_into()?);
let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address));
state
@@ -125,8 +127,8 @@ async fn delete_consumer_offset(
.delete_consumer_offset(
&session,
consumer,
- &query.stream_id,
- &query.topic_id,
+ &stream_id_ident,
+ &topic_id_ident,
query.partition_id,
)
.await
diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs
index f279b51be..48b2deb31 100644
--- a/core/server/src/http/http_shard_wrapper.rs
+++ b/core/server/src/http/http_shard_wrapper.rs
@@ -149,11 +149,7 @@ impl HttpSafeShard {
future.await
}
- pub async fn create_stream(
- &self,
- session: &Session,
- name: String,
- ) -> Result<crate::streaming::streams::stream::Stream, IggyError> {
+ pub async fn create_stream(&self, session: &Session, name: String) -> Result<usize, IggyError> {
let future = SendWrapper::new(self.shard().create_stream(session, name));
future.await
}
@@ -302,31 +298,40 @@ impl HttpSafeShard {
) -> Result<(), IggyError> {
self.shard().ensure_topic_exists(&stream_id, &topic_id)?;
- let partition_id = self.shard().streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- |(root, auxilary, ..)| match partitioning.kind {
- PartitioningKind::Balanced => {
- let upperbound = root.partitions().len();
- let pid = auxilary.get_next_partition_id(upperbound);
- Ok(pid)
- }
- PartitioningKind::PartitionId => Ok(u32::from_le_bytes(
- partitioning.value[..partitioning.length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- ) as usize),
- PartitioningKind::MessagesKey => {
- let upperbound = root.partitions().len();
- Ok(
- topics::helpers::calculate_partition_id_by_messages_key_hash(
- upperbound,
- &partitioning.value,
- ),
- )
- }
- },
- )?;
+ // Get stream/topic IDs and partition count from SharedMetadata
+ let numeric_stream_id = self
+ .shard()
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream existence already verified");
+ let numeric_topic_id = self
+ .shard()
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic existence already verified");
+ let partitions_count = self
+ .shard()
+ .shared_metadata
+ .partitions_count(numeric_stream_id, numeric_topic_id);
+
+ let partition_id = match partitioning.kind {
+ PartitioningKind::Balanced => self.shard().shared_metadata.get_next_partition_id(
+ numeric_stream_id,
+ numeric_topic_id,
+ partitions_count,
+ ),
+ PartitioningKind::PartitionId => u32::from_le_bytes(
+ partitioning.value[..partitioning.length as usize]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ) as usize,
+ PartitioningKind::MessagesKey => {
+ topics::helpers::calculate_partition_id_by_messages_key_hash(
+ partitions_count,
+ &partitioning.value,
+ )
+ }
+ };
let future = SendWrapper::new(self.shard().append_messages(
user_id,
diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs
index d2910ae1e..5961d42fc 100644
--- a/core/server/src/http/mapper.rs
+++ b/core/server/src/http/mapper.rs
@@ -17,95 +17,18 @@
*/
use crate::http::jwt::json_web_token::GeneratedToken;
-use crate::slab::Keyed;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
+use crate::metadata::{ConsumerGroupMeta, GlobalMetadata, PartitionMeta, StreamMeta, TopicMeta};
+use crate::shard::namespace::IggyNamespace;
use crate::streaming::clients::client_manager::Client;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
-use crate::streaming::stats::TopicStats;
-use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot};
-use crate::streaming::topics::topic::TopicRoot;
use crate::streaming::users::user::User;
use iggy_common::{ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, IggyByteSize};
use iggy_common::{IdentityInfo, PersonalAccessTokenInfo, TokenInfo, TopicDetails};
use iggy_common::{UserInfo, UserInfoDetails};
-use slab::Slab;
-use std::sync::Arc;
-
-/// Map TopicRoot with partitions to TopicDetails for HTTP responses
-pub fn map_topic_details(root: &TopicRoot, stats: &TopicStats) -> TopicDetails {
- let mut partitions = Vec::new();
-
- // Get partition details similar to binary mapper
- root.partitions().with_components(|partition_components| {
- let (partition_roots, partition_stats, _, offsets, _, _, _) =
- partition_components.into_components();
- for (partition_root, partition_stat, offset) in partition_roots
- .iter()
- .map(|(_, val)| val)
- .zip(partition_stats.iter().map(|(_, val)| val))
- .zip(offsets.iter().map(|(_, val)| val))
- .map(|((root, stat), offset)| (root, stat, offset))
- {
- partitions.push(iggy_common::Partition {
- id: partition_root.id() as u32,
- created_at: partition_root.created_at(),
- segments_count: partition_stat.segments_count_inconsistent(),
- current_offset: offset.load(std::sync::atomic::Ordering::Relaxed),
- size: IggyByteSize::from(partition_stat.size_bytes_inconsistent()),
- messages_count: partition_stat.messages_count_inconsistent(),
- });
- }
- });
-
- // Sort partitions by ID
- partitions.sort_by(|a, b| a.id.cmp(&b.id));
-
- TopicDetails {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
- partitions_count: partitions.len() as u32,
- partitions,
- message_expiry: root.message_expiry(),
- compression_algorithm: root.compression_algorithm(),
- max_topic_size: root.max_topic_size(),
- replication_factor: root.replication_factor(),
- }
-}
-
-/// Map TopicRoot and TopicStats to Topic for HTTP responses
-pub fn map_topic(root: &TopicRoot, stats: &TopicStats) -> iggy_common::Topic {
- iggy_common::Topic {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- size: stats.size_bytes_inconsistent().into(),
- partitions_count: root.partitions().len() as u32,
- messages_count: stats.messages_count_inconsistent(),
- message_expiry: root.message_expiry(),
- compression_algorithm: root.compression_algorithm(),
- max_topic_size: root.max_topic_size(),
- replication_factor: root.replication_factor(),
- }
-}
-/// Map multiple topics from slab components to Vec<Topic> for HTTP responses
-pub fn map_topics_from_components(
- roots: &Slab<TopicRoot>,
- stats: &Slab<Arc<TopicStats>>,
-) -> Vec<iggy_common::Topic> {
- let mut topics = roots
- .iter()
- .map(|(_, root)| root)
- .zip(stats.iter().map(|(_, stat)| stat))
- .map(|(root, stat)| map_topic(root, stat))
- .collect::<Vec<_>>();
-
- topics.sort_by(|a, b| a.id.cmp(&b.id));
- topics
-}
+// =====================================================================
+// User mapping functions
+// =====================================================================
pub fn map_user(user: &User) -> UserInfoDetails {
UserInfoDetails {
@@ -132,6 +55,10 @@ pub fn map_users(users: &[&User]) -> Vec<UserInfo> {
users_data
}
+// =====================================================================
+// Personal access token mapping functions
+// =====================================================================
+
pub fn map_personal_access_tokens(
personal_access_tokens: &[PersonalAccessToken],
) -> Vec<PersonalAccessTokenInfo> {
@@ -147,6 +74,10 @@ pub fn map_personal_access_tokens(
personal_access_tokens_data
}
+// =====================================================================
+// Client mapping functions
+// =====================================================================
+
pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails {
iggy_common::ClientInfoDetails {
client_id: client.session.client_id,
@@ -183,51 +114,9 @@ pub fn map_clients(clients: &[Client]) -> Vec<iggy_common::ClientInfo> {
all_clients
}
-pub fn map_consumer_groups(
- roots: &slab::Slab<ConsumerGroupRoot>,
- members: &slab::Slab<ConsumerGroupMembers>,
-) -> Vec<iggy_common::ConsumerGroup> {
- let mut groups = Vec::new();
- for (root, member) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(members.iter().map(|(_, val)| val))
- {
- let members_guard = member.inner().shared_get();
- let consumer_group = iggy_common::ConsumerGroup {
- id: root.id() as u32,
- name: root.key().clone(),
- partitions_count: root.partitions().len() as u32,
- members_count: members_guard.len() as u32,
- };
- groups.push(consumer_group);
- }
- groups.sort_by(|a, b| a.id.cmp(&b.id));
- groups
-}
-
-pub fn map_consumer_group(
- root: &ConsumerGroupRoot,
- members: &ConsumerGroupMembers,
-) -> ConsumerGroupDetails {
- let members_guard = members.inner().shared_get();
- let mut consumer_group_details = ConsumerGroupDetails {
- id: root.id() as u32,
- name: root.key().clone(),
- partitions_count: root.partitions().len() as u32,
- members_count: members_guard.len() as u32,
- members: Vec::new(),
- };
-
- for (_, member) in members_guard.iter() {
- consumer_group_details.members.push(ConsumerGroupMember {
- id: member.id as u32,
- partitions_count: member.partitions.len() as u32,
- partitions: member.partitions.iter().map(|p| *p as u32).collect(),
- });
- }
- consumer_group_details
-}
+// =====================================================================
+// Token mapping functions
+// =====================================================================
pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> IdentityInfo {
IdentityInfo {
@@ -239,69 +128,294 @@ pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> Ide
}
}
-/// Map StreamRoot and StreamStats to StreamDetails for HTTP responses
-pub fn map_stream_details(
- root: &crate::streaming::streams::stream::StreamRoot,
- stats: &crate::streaming::stats::StreamStats,
+// =====================================================================
+// SharedMetadata-based mapping functions
+// =====================================================================
+
+/// Map a stream from SharedMetadata to StreamDetails (with topics)
+pub fn map_stream_details_from_metadata(
+ stream_meta: &StreamMeta,
+ metadata: &GlobalMetadata,
) -> iggy_common::StreamDetails {
- // Get topics using the new slab-based API
- let topics = root.topics().with_components(|topic_ref| {
- let (topic_roots, _topic_auxiliaries, topic_stats) = topic_ref.into_components();
- let mut topics_vec = Vec::new();
+ let stream_id = stream_meta.id;
- // Iterate over topics in the stream
- for (topic_root, topic_stat) in topic_roots
- .iter()
- .map(|(_, root)| root)
- .zip(topic_stats.iter().map(|(_, stat)| stat))
- {
- topics_vec.push(map_topic(topic_root, topic_stat));
+ // Get topic IDs for this stream (sorted)
+ let mut topic_ids: Vec<_> = metadata
+ .topics_by_stream
+ .get(&stream_id)
+ .map(|ids| ids.iter().copied().collect())
+ .unwrap_or_default();
+ topic_ids.sort_unstable();
+
+ // Map topics
+ let mut topics = Vec::with_capacity(topic_ids.len());
+ for &topic_id in &topic_ids {
+ if let Some(topic_meta) = metadata.topics.get(&(stream_id, topic_id)) {
+ topics.push(map_topic_from_metadata(topic_meta, metadata));
}
+ }
- // Sort topics by ID for consistent ordering
- topics_vec.sort_by(|a, b| a.id.cmp(&b.id));
- topics_vec
- });
+ // Aggregate stats
+ let (total_size, total_messages) = aggregate_stream_stats(stream_id, metadata);
iggy_common::StreamDetails {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- topics_count: root.topics_count() as u32,
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
+ id: stream_meta.id as u32,
+ created_at: stream_meta.created_at,
+ name: stream_meta.name.to_string(),
+ topics_count: topics.len() as u32,
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
topics,
}
}
-/// Map StreamRoot and StreamStats to Stream for HTTP responses
-pub fn map_stream(
- root: &crate::streaming::streams::stream::StreamRoot,
- stats: &crate::streaming::stats::StreamStats,
+/// Map a stream from SharedMetadata to Stream (without topics)
+pub fn map_stream_from_metadata(
+ stream_meta: &StreamMeta,
+ metadata: &GlobalMetadata,
) -> iggy_common::Stream {
+ let (total_size, total_messages) = aggregate_stream_stats(stream_meta.id, metadata);
+
+ let topics_count = metadata
+ .topics_by_stream
+ .get(&stream_meta.id)
+ .map(|ids| ids.len())
+ .unwrap_or(0);
+
iggy_common::Stream {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- topics_count: root.topics_count() as u32,
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
+ id: stream_meta.id as u32,
+ created_at: stream_meta.created_at,
+ name: stream_meta.name.to_string(),
+ topics_count: topics_count as u32,
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
}
}
-/// Map multiple streams from slabs
-pub fn map_streams_from_slabs(
- roots: &slab::Slab<crate::streaming::streams::stream::StreamRoot>,
- stats: &slab::Slab<Arc<crate::streaming::stats::StreamStats>>,
-) -> Vec<iggy_common::Stream> {
- let mut streams = Vec::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- streams.push(map_stream(root, stat));
+/// Map all streams from SharedMetadata
+pub fn map_streams_from_metadata(metadata: &GlobalMetadata) -> Vec<iggy_common::Stream> {
+ let mut stream_ids: Vec<_> = metadata.streams.keys().copied().collect();
+ stream_ids.sort_unstable();
+
+ let mut streams = Vec::with_capacity(stream_ids.len());
+ for stream_id in stream_ids {
+ if let Some(stream_meta) = metadata.streams.get(&stream_id) {
+ streams.push(map_stream_from_metadata(stream_meta, metadata));
+ }
}
- streams.sort_by(|a, b| a.id.cmp(&b.id));
streams
}
+
+/// Map a topic from SharedMetadata to Topic (without partitions)
+pub fn map_topic_from_metadata(
+ topic_meta: &TopicMeta,
+ metadata: &GlobalMetadata,
+) -> iggy_common::Topic {
+ let (total_size, total_messages) =
+ aggregate_topic_stats(topic_meta.stream_id, topic_meta.id, metadata);
+
+ let partitions_count = metadata
+ .partitions_by_topic
+ .get(&(topic_meta.stream_id, topic_meta.id))
+ .map(|ids| ids.len())
+ .unwrap_or(0);
+
+ iggy_common::Topic {
+ id: topic_meta.id as u32,
+ created_at: topic_meta.created_at,
+ name: topic_meta.name.to_string(),
+ size: IggyByteSize::from(total_size),
+ partitions_count: partitions_count as u32,
+ messages_count: total_messages,
+ message_expiry: topic_meta.message_expiry,
+ compression_algorithm: topic_meta.compression_algorithm,
+ max_topic_size: topic_meta.max_topic_size,
+ replication_factor: topic_meta.replication_factor,
+ }
+}
+
+/// Map all topics for a stream from SharedMetadata
+pub fn map_topics_from_metadata(
+ stream_id: usize,
+ metadata: &GlobalMetadata,
+) -> Vec<iggy_common::Topic> {
+ let mut topic_ids: Vec<_> = metadata
+ .topics_by_stream
+ .get(&stream_id)
+ .map(|ids| ids.iter().copied().collect())
+ .unwrap_or_default();
+ topic_ids.sort_unstable();
+
+ let mut topics = Vec::with_capacity(topic_ids.len());
+ for topic_id in topic_ids {
+ if let Some(topic_meta) = metadata.topics.get(&(stream_id, topic_id)) {
+ topics.push(map_topic_from_metadata(topic_meta, metadata));
+ }
+ }
+ topics
+}
+
+/// Map a topic from SharedMetadata to TopicDetails (with partitions)
+pub fn map_topic_details_from_metadata(
+ topic_meta: &TopicMeta,
+ metadata: &GlobalMetadata,
+) -> TopicDetails {
+ let stream_id = topic_meta.stream_id;
+ let topic_id = topic_meta.id;
+
+ // Get partition IDs (sorted)
+ let mut partition_ids: Vec<_> = metadata
+ .partitions_by_topic
+ .get(&(stream_id, topic_id))
+ .map(|ids| ids.iter().copied().collect())
+ .unwrap_or_default();
+ partition_ids.sort_unstable();
+
+ // Map partitions
+ let mut partitions = Vec::with_capacity(partition_ids.len());
+ for &partition_id in &partition_ids {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ if let Some(partition_meta) = metadata.partitions.get(&ns) {
+ partitions.push(map_partition_from_metadata(partition_meta));
+ }
+ }
+
+ // Aggregate stats
+ let (total_size, total_messages) = aggregate_topic_stats(stream_id, topic_id, metadata);
+
+ TopicDetails {
+ id: topic_meta.id as u32,
+ created_at: topic_meta.created_at,
+ name: topic_meta.name.to_string(),
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
+ partitions_count: partitions.len() as u32,
+ partitions,
+ message_expiry: topic_meta.message_expiry,
+ compression_algorithm: topic_meta.compression_algorithm,
+ max_topic_size: topic_meta.max_topic_size,
+ replication_factor: topic_meta.replication_factor,
+ }
+}
+
+/// Map a partition from SharedMetadata
+pub fn map_partition_from_metadata(partition_meta: &PartitionMeta) -> iggy_common::Partition {
+ let stats = &partition_meta.stats;
+ let segments_count = stats.segments_count_inconsistent();
+ let size_bytes = stats.size_bytes_inconsistent();
+ let messages_count = stats.messages_count_inconsistent();
+ let current_offset = stats.current_offset();
+
+ iggy_common::Partition {
+ id: partition_meta.id as u32,
+ created_at: partition_meta.created_at,
+ segments_count,
+ current_offset,
+ size: IggyByteSize::from(size_bytes),
+ messages_count,
+ }
+}
+
+/// Map a consumer group from SharedMetadata
+pub fn map_consumer_group_from_metadata(cg_meta: &ConsumerGroupMeta) -> iggy_common::ConsumerGroup {
+ iggy_common::ConsumerGroup {
+ id: cg_meta.id as u32,
+ name: cg_meta.name.to_string(),
+ partitions_count: cg_meta.partitions.len() as u32,
+ members_count: cg_meta.members.len() as u32,
+ }
+}
+
+/// Map a consumer group to ConsumerGroupDetails from SharedMetadata
+pub fn map_consumer_group_details_from_metadata(
+ cg_meta: &ConsumerGroupMeta,
+) -> ConsumerGroupDetails {
+ let members: Vec<ConsumerGroupMember> = cg_meta
+ .members
+ .iter()
+ .map(|m| ConsumerGroupMember {
+ id: m.id as u32,
+ partitions_count: m.partitions.len() as u32,
+ partitions: m.partitions.iter().map(|&p| p as u32).collect(),
+ })
+ .collect();
+
+ ConsumerGroupDetails {
+ id: cg_meta.id as u32,
+ name: cg_meta.name.to_string(),
+ partitions_count: cg_meta.partitions.len() as u32,
+ members_count: members.len() as u32,
+ members,
+ }
+}
+
+/// Map all consumer groups for a topic from SharedMetadata
+pub fn map_consumer_groups_from_metadata(
+ stream_id: usize,
+ topic_id: usize,
+ metadata: &GlobalMetadata,
+) -> Vec<iggy_common::ConsumerGroup> {
+ let mut group_ids: Vec<_> = metadata
+ .consumer_groups_by_topic
+ .get(&(stream_id, topic_id))
+ .map(|ids| ids.iter().copied().collect())
+ .unwrap_or_default();
+ group_ids.sort_unstable();
+
+ let mut groups = Vec::with_capacity(group_ids.len());
+ for group_id in group_ids {
+ if let Some(cg_meta) = metadata
+ .consumer_groups
+ .get(&(stream_id, topic_id, group_id))
+ {
+ groups.push(map_consumer_group_from_metadata(cg_meta));
+ }
+ }
+ groups
+}
+
+// =====================================================================
+// Helper functions for stats aggregation
+// =====================================================================
+
+fn aggregate_stream_stats(stream_id: usize, metadata: &GlobalMetadata) -> (u64, u64) {
+ let mut total_size = 0u64;
+ let mut total_messages = 0u64;
+
+ if let Some(topic_ids) = metadata.topics_by_stream.get(&stream_id) {
+ for &topic_id in topic_ids.iter() {
+ if let Some(partition_ids) = metadata.partitions_by_topic.get(&(stream_id, topic_id)) {
+ for &partition_id in partition_ids.iter() {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ if let Some(partition_meta) = metadata.partitions.get(&ns) {
+ total_size += partition_meta.stats.size_bytes_inconsistent();
+ total_messages += partition_meta.stats.messages_count_inconsistent();
+ }
+ }
+ }
+ }
+ }
+
+ (total_size, total_messages)
+}
+
+fn aggregate_topic_stats(
+ stream_id: usize,
+ topic_id: usize,
+ metadata: &GlobalMetadata,
+) -> (u64, u64) {
+ let mut total_size = 0u64;
+ let mut total_messages = 0u64;
+
+ if let Some(partition_ids) = metadata.partitions_by_topic.get(&(stream_id, topic_id)) {
+ for &partition_id in partition_ids.iter() {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ if let Some(partition_meta) = metadata.partitions.get(&ns) {
+ total_size += partition_meta.stats.size_bytes_inconsistent();
+ total_messages += partition_meta.stats.messages_count_inconsistent();
+ }
+ }
+ }
+
+ (total_size, total_messages)
+}
diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs
index 959e1fd2c..10f3f27a4 100644
--- a/core/server/src/http/partitions.rs
+++ b/core/server/src/http/partitions.rs
@@ -60,7 +60,7 @@ async fn create_partitions(
let _parititon_guard = state.shard.shard().fs_locks.partition_lock.lock().await;
let session = Session::stateless(identity.user_id, identity.ip_address);
- let partitions = SendWrapper::new(state.shard.shard().create_partitions(
+ let partition_infos = SendWrapper::new(state.shard.shard().create_partitions(
&session,
&command.stream_id,
&command.topic_id,
@@ -74,7 +74,7 @@ async fn create_partitions(
let event = ShardEvent::CreatedPartitions {
stream_id: command.stream_id.clone(),
topic_id: command.topic_id.clone(),
- partitions,
+ partitions: partition_infos,
};
let _responses = shard.broadcast_event_to_all_shards(event).await;
Ok::<(), CustomError>(())
diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs
index 9054777ea..1223d02ad 100644
--- a/core/server/src/http/streams.rs
+++ b/core/server/src/http/streams.rs
@@ -20,7 +20,6 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
use axum::extract::{Path, State};
use axum::http::StatusCode;
@@ -58,34 +57,32 @@ async fn get_stream(
Path(stream_id): Path<String>,
) -> Result<Json<StreamDetails>, CustomError> {
let stream_id = Identifier::from_str_value(&stream_id)?;
- let exists = state.shard.shard().ensure_stream_exists(&stream_id).is_ok();
- if !exists {
- return Err(CustomError::ResourceNotFound);
- }
- // Use direct slab access for thread-safe stream retrieval
- let stream_details = SendWrapper::new(|| {
- state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&stream_id, |(root, stats)| {
- crate::http::mapper::map_stream_details(&root, &stats)
- })
- })();
+ // Use SharedMetadata for stream retrieval
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let metadata = shard.shared_metadata.load();
+ let stream_meta = metadata
+ .streams
+ .get(&numeric_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let stream_details =
+ crate::http::mapper::map_stream_details_from_metadata(stream_meta, &metadata);
Ok(Json(stream_details))
}
#[debug_handler]
async fn get_streams(State(state): State<Arc<AppState>>) -> Result<Json<Vec<Stream>>, CustomError> {
- // Use direct slab access for thread-safe streams retrieval
- let streams = SendWrapper::new(|| {
- state.shard.shard().streams.with_components(|stream_ref| {
- let (roots, stats) = stream_ref.into_components();
- crate::http::mapper::map_streams_from_slabs(&roots, &stats)
- })
- })();
+ // Use SharedMetadata for streams retrieval
+ let shard = state.shard.shard();
+ let metadata = shard.shared_metadata.load();
+ let streams = crate::http::mapper::map_streams_from_metadata(&metadata);
Ok(Json(streams))
}
@@ -104,7 +101,7 @@ async fn create_stream(
let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
// Create stream using wrapper method
- let stream = state
+ let created_stream_id = state
.shard
.create_stream(&session, command.name.clone())
.await
@@ -115,14 +112,11 @@ async fn create_stream(
)
})?;
- let created_stream_id = stream.root().id();
-
// Send event for stream creation - inlined from wrapper
{
use crate::shard::transmission::event::ShardEvent;
let event = ShardEvent::CreatedStream {
id: created_stream_id,
- stream,
};
let _responses = state
.shard
@@ -148,16 +142,15 @@ async fn create_stream(
)
})?;
- // Get the created stream details using direct slab access
- let response = SendWrapper::new(|| {
- state
- .shard
- .shard()
- .streams
- .with_components_by_id(created_stream_id, |(root, stats)| {
- crate::http::mapper::map_stream_details(&root, &stats)
- })
- })();
+ // Get the created stream details from SharedMetadata
+ let shard = state.shard.shard();
+ let metadata = shard.shared_metadata.load();
+ let stream_meta = metadata
+ .streams
+ .get(&created_stream_id)
+ .expect("Stream must exist after creation");
+ let response =
+ crate::http::mapper::map_stream_details_from_metadata(stream_meta, &metadata);
Ok::<Json<StreamDetails>, CustomError>(Json(response))
});
@@ -233,8 +226,8 @@ async fn delete_stream(
let session = Session::stateless(identity.user_id, identity.ip_address);
let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
- // Delete stream and get the stream entity
- let stream = {
+ // Delete stream and get the stream info
+ let stream_info = {
let future = SendWrapper::new(
state
.shard
@@ -247,7 +240,7 @@ async fn delete_stream(
format!("{COMPONENT} (error: {e}) - failed to delete stream with ID: {stream_id}",)
})?;
- let stream_id_numeric = stream.root().id();
+ let stream_id_numeric = stream_info.id;
// Send event for stream deletion
{
diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs
index 8341c2cfa..cad4a1b72 100644
--- a/core/server/src/http/topics.rs
+++ b/core/server/src/http/topics.rs
@@ -20,11 +20,9 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents};
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get};
@@ -68,41 +66,25 @@ async fn get_topic(
let identity_topic_id = Identifier::from_str_value(&topic_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
+ let shard = state.shard.shard();
- // Check permissions and stream existence
- state.shard.shard().ensure_authenticated(&session)?;
- let stream_exists = state
- .shard
- .shard()
- .ensure_stream_exists(&identity_stream_id)
- .is_ok();
- if !stream_exists {
- return Err(CustomError::ResourceNotFound);
- }
+ // Check authentication
+ shard.ensure_authenticated(&session)?;
- let topic_exists = state
- .shard
- .shard()
- .ensure_topic_exists(&identity_stream_id, &identity_topic_id)
- .is_ok();
- if !topic_exists {
- return Err(CustomError::ResourceNotFound);
- }
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&identity_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
- let numeric_stream_id = state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&identity_stream_id, streams::helpers::get_stream_id());
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identity_stream_id,
- &identity_topic_id,
- topics::helpers::get_topic_id(),
- );
-
- state.shard.shard()
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &identity_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ // Check permissions
+ shard
.permissioner
- .borrow()
.get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.error(|e: &IggyError| {
format!(
@@ -111,12 +93,14 @@ async fn get_topic(
)
})?;
- // Get topic details using the new API
- let topic_details = state.shard.shard().streams.with_topic_by_id(
- &identity_stream_id,
- &identity_topic_id,
- |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats),
- );
+ // Get topic details from SharedMetadata
+ let metadata = shard.shared_metadata.load();
+ let topic_meta = metadata
+ .topics
+ .get(&(numeric_stream_id, numeric_topic_id))
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let topic_details = crate::http::mapper::map_topic_details_from_metadata(topic_meta, &metadata);
Ok(Json(topic_details))
}
@@ -127,22 +111,22 @@ async fn get_topics(
Extension(identity): Extension<Identity>,
Path(stream_id): Path<String>,
) -> Result<Json<Vec<Topic>>, CustomError> {
- let stream_id = Identifier::from_str_value(&stream_id)?;
+ let stream_id_ident = Identifier::from_str_value(&stream_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
+ let shard = state.shard.shard();
- // Check permissions and stream existence
- state.shard.shard().ensure_authenticated(&session)?;
- state.shard.shard().ensure_stream_exists(&stream_id)?;
+ // Check authentication
+ shard.ensure_authenticated(&session)?;
- let numeric_stream_id = state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id_ident)
+ .ok_or(CustomError::ResourceNotFound)?;
- state.shard.shard()
+ // Check permissions
+ shard
.permissioner
- .borrow()
.get_topics(session.get_user_id(), numeric_stream_id)
.error(|e: &IggyError| {
format!(
@@ -151,17 +135,9 @@ async fn get_topics(
)
})?;
- // Get topics using the new API
- let topics = state
- .shard
- .shard()
- .streams
- .with_topics(&stream_id, |topics| {
- topics.with_components(|topics| {
- let (roots, _, stats) = topics.into_components();
- crate::http::mapper::map_topics_from_components(&roots, &stats)
- })
- });
+ // Get topics from SharedMetadata
+ let metadata = shard.shared_metadata.load();
+ let topics = crate::http::mapper::map_topics_from_metadata(numeric_stream_id, &metadata);
Ok(Json(topics))
}
@@ -180,7 +156,7 @@ async fn create_topic(
let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address));
let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
- let topic = {
+ let topic_id = {
let future = SendWrapper::new(state.shard.shard().create_topic(
&session,
&command.stream_id,
@@ -196,11 +172,19 @@ async fn create_topic(
format!("{COMPONENT} (error: {e}) - failed to create topic, stream ID: {stream_id}")
})?;
- // Update command with actual values from created topic
- command.message_expiry = topic.root().message_expiry();
- command.max_topic_size = topic.root().max_topic_size();
-
- let topic_id = topic.id();
+ // Update command with actual values from created topic via SharedMetadata
+ {
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+ let metadata = shard.shared_metadata.load();
+ if let Some(topic_meta) = metadata.topics.get(&(numeric_stream_id, topic_id)) {
+ command.message_expiry = topic_meta.message_expiry;
+ command.max_topic_size = topic_meta.max_topic_size;
+ }
+ }
// Send events for topic creation
let broadcast_future = SendWrapper::new(async {
@@ -210,12 +194,12 @@ async fn create_topic(
let event = ShardEvent::CreatedTopic {
stream_id: command.stream_id.clone(),
- topic,
+ topic_id,
};
let _responses = shard.broadcast_event_to_all_shards(event).await;
// Create partitions
- let partitions = shard
+ let partition_infos = shard
.create_partitions(
&session,
&command.stream_id,
@@ -227,7 +211,7 @@ async fn create_topic(
let event = ShardEvent::CreatedPartitions {
stream_id: command.stream_id.clone(),
topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions,
+ partitions: partition_infos,
};
let _responses = shard.broadcast_event_to_all_shards(event).await;
@@ -240,14 +224,20 @@ async fn create_topic(
)
})?;
- // Create response using the same approach as binary handler
+ // Create response from SharedMetadata
let response = {
- let topic_identifier = Identifier::numeric(topic_id as u32).unwrap();
- let topic_response = state.shard.shard().streams.with_topic_by_id(
- &command.stream_id,
- &topic_identifier,
- |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats),
- );
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+ let metadata = shard.shared_metadata.load();
+ let topic_meta = metadata
+ .topics
+ .get(&(numeric_stream_id, topic_id))
+ .expect("Topic must exist after creation");
+ let topic_response =
+ crate::http::mapper::map_topic_details_from_metadata(topic_meta, &metadata);
Json(topic_response)
};
@@ -303,19 +293,31 @@ async fn update_topic(
)
})?;
- // TODO: Tech debt.
- let topic_id = if name_changed {
+ // Get the updated values from SharedMetadata
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+
+ // Use the new name if it was changed, otherwise use the original topic_id
+ let topic_id_for_lookup = if name_changed {
Identifier::named(&command.name.clone()).unwrap()
} else {
command.topic_id.clone()
};
- // Get the updated values from the topic
- let (message_expiry, max_topic_size) = state.shard.shard().streams.with_topic_by_id(
- &command.stream_id,
- &topic_id,
- |(root, _, _)| (root.message_expiry(), root.max_topic_size()),
- );
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id_for_lookup)
+ .expect("Topic must exist after update");
+
+ let metadata = shard.shared_metadata.load();
+ let topic_meta = metadata
+ .topics
+ .get(&(numeric_stream_id, numeric_topic_id))
+ .expect("Topic metadata must exist");
+ let (message_expiry, max_topic_size) = (topic_meta.message_expiry, topic_meta.max_topic_size);
// Send event for topic update
{
@@ -369,7 +371,7 @@ async fn delete_topic(
let session = Session::stateless(identity.user_id, identity.ip_address);
let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
- let topic = {
+ let topic_info = {
let future = SendWrapper::new(state.shard.shard().delete_topic(
&session,
&identifier_stream_id,
@@ -382,7 +384,7 @@ async fn delete_topic(
)
})?;
- let topic_id_numeric = topic.root().id();
+ let topic_id_numeric = topic_info.id;
// Send event for topic deletion
{
diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs
index 29ae19230..47110dfb9 100644
--- a/core/server/src/http/users.rs
+++ b/core/server/src/http/users.rs
@@ -137,26 +137,6 @@ async fn create_user(
let user_id = user.id;
let response = Json(mapper::map_user(&user));
- // Send event for user creation
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::CreatedUser {
- user_id,
- username: command.username.to_owned(),
- password: command.password.to_owned(),
- status: command.status,
- permissions: command.permissions.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let username = command.username.clone();
let entry_command = EntryCommand::CreateUser(CreateUserWithId {
@@ -212,24 +192,6 @@ async fn update_user(
format!("{COMPONENT} (error: {e}) - failed to update user, user ID: {user_id}")
})?;
- // Send event for user update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedUser {
- user_id: command.user_id.clone(),
- username: command.username.clone(),
- status: command.status,
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let username = command.username.clone();
let entry_command = EntryCommand::UpdateUser(command);
@@ -271,23 +233,6 @@ async fn update_permissions(
format!("{COMPONENT} (error: {e}) - failed to update permissions, user ID: {user_id}")
})?;
- // Send event for permissions update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedPermissions {
- user_id: command.user_id.clone(),
- permissions: command.permissions.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::UpdatePermissions(command);
let future = SendWrapper::new(
@@ -332,24 +277,6 @@ async fn change_password(
format!("{COMPONENT} (error: {e}) - failed to change password, user ID: {user_id}")
})?;
- // Send event for password change
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::ChangedPassword {
- user_id: command.user_id.clone(),
- current_password: command.current_password.clone(),
- new_password: command.new_password.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::ChangePassword(command);
let future = SendWrapper::new(
@@ -388,22 +315,6 @@ async fn delete_user(
format!("{COMPONENT} (error: {e}) - failed to delete user with ID: {user_id}")
})?;
- // Send event for user deletion
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::DeletedUser {
- user_id: identifier_user_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::DeleteUser(DeleteUser {
user_id: identifier_user_id,
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 2919214b6..b6a758006 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -37,10 +37,10 @@ pub mod diagnostics;
pub mod http;
pub mod io;
pub mod log;
+pub mod metadata;
pub mod quic;
pub mod server_error;
pub mod shard;
-pub mod slab;
pub mod state;
pub mod streaming;
pub mod tcp;
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3fe36e20d..d034ba5f9 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -28,19 +28,17 @@ use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError, MemoryPool};
use server::SEMANTIC_VERSION;
use server::args::Args;
use server::bootstrap::{
- create_directories, create_shard_connections, create_shard_executor, load_config, load_streams,
- load_users, resolve_persister, update_system_info,
+ create_directories, create_shard_connections, create_shard_executor, load_config,
+ load_metadata_only, load_users_to_metadata, resolve_persister, update_system_info,
};
use server::configs::sharding::ShardAllocator;
use server::diagnostics::{print_io_uring_permission_info, print_locked_memory_limit_info};
use server::io::fs_utils;
use server::log::logger::Logging;
+use server::metadata::{SharedConsumerOffsetsStore, SharedMetadata};
use server::server_error::ServerError;
use server::shard::system::info::SystemInfo;
use server::shard::{IggyShard, calculate_shard_assignment};
-use server::slab::traits_ext::{
- EntityComponentSystem, EntityComponentSystemMutCell, IntoComponents,
-};
use server::state::file::FileState;
use server::state::system::SystemState;
use server::streaming::clients::client_manager::{Client, ClientManager};
@@ -278,8 +276,16 @@ fn main() -> Result<(), ServerError> {
);
let state = SystemState::load(state).await?;
let (streams_state, users_state) = state.decompose();
- let streams = load_streams(streams_state.into_values(), &config.system).await?;
- let users = load_users(users_state.into_values());
+
+ // Create shared metadata structures (leaked for 'static lifetime)
+ let shared_metadata: &'static SharedMetadata =
+ Box::leak(Box::new(SharedMetadata::default()));
+ let shared_consumer_offsets: &'static SharedConsumerOffsetsStore =
+ Box::leak(Box::new(SharedConsumerOffsetsStore::new()));
+
+ // Load metadata directly into SharedMetadata (no slabs)
+ load_metadata_only(streams_state.into_values(), shared_metadata);
+ load_users_to_metadata(users_state.into_values(), shared_metadata);
// ELEVENTH DISCRETE LOADING STEP.
let shard_allocator = ShardAllocator::new(&config.system.sharding.cpu_allocation)?;
@@ -320,41 +326,26 @@ fn main() -> Result<(), ServerError> {
let client_manager: EternalPtr<DashMap<u32, Client>> = client_manager.into();
let client_manager = ClientManager::new(client_manager);
- streams.with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, stream) in root.iter() {
- stream.topics().with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, topic) in root.iter() {
- topic.partitions().with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, partition) in root.iter() {
- let stream_id = stream.id();
- let topic_id = topic.id();
- let partition_id = partition.id();
- let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
- let shard_id = ShardId::new(calculate_shard_assignment(
- &ns,
- shard_assignment.len() as u32,
- ));
- // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integratedds
- let location = PartitionLocation::new(shard_id, LocalIdx::new(0));
- shards_table.insert(ns, location);
- }
- });
- }
- })
+ // Populate shards_table from SharedMetadata partitions
+ {
+ let metadata = shared_metadata.load();
+ for (ns, _partition_meta) in metadata.partitions.iter() {
+ let shard_id = ShardId::new(calculate_shard_assignment(
+ ns,
+ shard_assignment.len() as u32,
+ ));
+ // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integrated
+ let location = PartitionLocation::new(shard_id, LocalIdx::new(0));
+ shards_table.insert(*ns, location);
}
- });
+ }
for (id, assignment) in shard_assignment
.into_iter()
.enumerate()
.map(|(idx, assignment)| (idx as u16, assignment))
{
- let streams = streams.clone();
let shards_table = shards_table.clone();
- let users = users.clone();
let connections = connections.clone();
let config = config.clone();
let encryptor = encryptor.clone();
@@ -373,29 +364,6 @@ fn main() -> Result<(), ServerError> {
);
let client_manager = client_manager.clone();
- // TODO: Explore decoupling the `Log` from `Partition` entity.
- // Ergh... I knew this will backfire to include `Log` as part of the `Partition` entity,
- // We have to initialize with a default log for every partition, once we `Clone` the Streams / Topics / Partitions,
- // because `Clone` impl for `Partition` does not clone the actual log, just creates an empty one.
- streams.with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, stream) in root.iter() {
- stream.topics().with_components_mut(|components| {
- let (mut root, ..) = components.into_components();
- for (_, topic) in root.iter_mut() {
- let partitions_count = topic.partitions().len();
- for log_id in 0..partitions_count {
- let id = topic.partitions_mut().insert_default_log();
- assert_eq!(
- id, log_id,
- "main: partition_insert_default_log: id mismatch when creating default log"
- );
- }
- }
- })
- }
- });
-
let shard_done_tx = shard_done_tx.clone();
let handle = std::thread::Builder::new()
.name(format!("shard-{id}"))
@@ -414,9 +382,7 @@ fn main() -> Result<(), ServerError> {
let builder = IggyShard::builder();
let shard = builder
.id(id)
- .streams(streams)
.state(state)
- .users(users)
.shards_table(shards_table)
.connections(connections)
.clients_manager(client_manager)
@@ -425,6 +391,8 @@ fn main() -> Result<(), ServerError> {
.version(current_version)
.metrics(metrics)
.is_follower(is_follower)
+ .shared_metadata(shared_metadata)
+ .shared_consumer_offsets(shared_consumer_offsets)
.build();
let shard = Rc::new(shard);
diff --git a/core/server/src/metadata/consumer_group.rs b/core/server/src/metadata/consumer_group.rs
new file mode 100644
index 000000000..cd18c0d38
--- /dev/null
+++ b/core/server/src/metadata/consumer_group.rs
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{ClientId, ConsumerGroupId, PartitionId, StreamId, TopicId};
+use imbl::Vector as ImVector;
+use std::sync::Arc;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ConsumerGroupMemberMeta {
+ pub id: usize,
+ pub client_id: ClientId,
+ pub partitions: Vec<PartitionId>,
+}
+
+impl ConsumerGroupMemberMeta {
+ pub fn new(id: usize, client_id: ClientId) -> Self {
+ Self {
+ id,
+ client_id,
+ partitions: Vec::new(),
+ }
+ }
+}
+
+#[derive(Clone, Debug)]
+pub struct ConsumerGroupMeta {
+ pub id: ConsumerGroupId,
+ pub stream_id: StreamId,
+ pub topic_id: TopicId,
+ pub name: Arc<str>,
+ pub partitions: Vec<PartitionId>,
+ pub members: ImVector<ConsumerGroupMemberMeta>,
+}
+
+impl ConsumerGroupMeta {
+ /// Rebalance partition assignments among members (round-robin).
+ pub fn rebalance_members(&mut self) {
+ let partition_count = self.partitions.len();
+ let member_count = self.members.len();
+
+ if member_count == 0 || partition_count == 0 {
+ return;
+ }
+
+ for member in self.members.iter_mut() {
+ member.partitions.clear();
+ }
+
+ for (i, &partition_id) in self.partitions.iter().enumerate() {
+ let member_idx = i % member_count;
+ if let Some(member) = self.members.get_mut(member_idx) {
+ member.partitions.push(partition_id);
+ }
+ }
+ }
+}
diff --git a/core/server/src/metadata/consumer_offsets_store.rs b/core/server/src/metadata/consumer_offsets_store.rs
new file mode 100644
index 000000000..85db0735b
--- /dev/null
+++ b/core/server/src/metadata/consumer_offsets_store.rs
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{PartitionId, StreamId, TopicId};
+use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets};
+use std::sync::Arc;
+
+/// Cross-shard consumer offsets visibility using DashMap.
+///
+/// When a partition is created, the consumer offsets Arcs are registered here.
+/// Any shard can then access them without routing to the owning shard.
+#[derive(Debug, Default)]
+pub struct SharedConsumerOffsetsStore {
+ consumer_offsets: dashmap::DashMap<(StreamId, TopicId, PartitionId), Arc<ConsumerOffsets>>,
+ consumer_group_offsets:
+ dashmap::DashMap<(StreamId, TopicId, PartitionId), Arc<ConsumerGroupOffsets>>,
+}
+
+impl SharedConsumerOffsetsStore {
+ pub fn new() -> Self {
+ Self {
+ consumer_offsets: dashmap::DashMap::new(),
+ consumer_group_offsets: dashmap::DashMap::new(),
+ }
+ }
+
+ pub fn register_consumer_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ offsets: Arc<ConsumerOffsets>,
+ ) {
+ self.consumer_offsets
+ .insert((stream_id, topic_id, partition_id), offsets);
+ }
+
+ pub fn get_consumer_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ ) -> Option<Arc<ConsumerOffsets>> {
+ self.consumer_offsets
+ .get(&(stream_id, topic_id, partition_id))
+ .map(|r| Arc::clone(&r))
+ }
+
+ pub fn remove_consumer_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ ) {
+ self.consumer_offsets
+ .remove(&(stream_id, topic_id, partition_id));
+ }
+
+ pub fn register_consumer_group_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ offsets: Arc<ConsumerGroupOffsets>,
+ ) {
+ self.consumer_group_offsets
+ .insert((stream_id, topic_id, partition_id), offsets);
+ }
+
+ pub fn get_consumer_group_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ ) -> Option<Arc<ConsumerGroupOffsets>> {
+ self.consumer_group_offsets
+ .get(&(stream_id, topic_id, partition_id))
+ .map(|r| Arc::clone(&r))
+ }
+
+ pub fn remove_consumer_group_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ ) {
+ self.consumer_group_offsets
+ .remove(&(stream_id, topic_id, partition_id));
+ }
+
+ pub fn remove_partition_offsets(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_id: PartitionId,
+ ) {
+ self.remove_consumer_offsets(stream_id, topic_id, partition_id);
+ self.remove_consumer_group_offsets(stream_id, topic_id, partition_id);
+ }
+
+ pub fn remove_topic_offsets(&self, stream_id: StreamId, topic_id: TopicId) {
+ self.consumer_offsets
+ .retain(|&(s, t, _), _| s != stream_id || t != topic_id);
+ self.consumer_group_offsets
+ .retain(|&(s, t, _), _| s != stream_id || t != topic_id);
+ }
+
+ pub fn remove_stream_offsets(&self, stream_id: StreamId) {
+ self.consumer_offsets.retain(|&(s, _, _), _| s != stream_id);
+ self.consumer_group_offsets
+ .retain(|&(s, _, _), _| s != stream_id);
+ }
+}
diff --git a/core/server/src/metadata/mod.rs b/core/server/src/metadata/mod.rs
new file mode 100644
index 000000000..676635b60
--- /dev/null
+++ b/core/server/src/metadata/mod.rs
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared metadata module providing a single source of truth for all shards.
+//!
+//! This module provides an `ArcSwap`-based approach where all shards read from
+//! a shared snapshot, and only shard 0 can write (swap in new snapshots).
+//!
+//! # Architecture
+//!
+//! - `GlobalMetadata` (snapshot.rs): Immutable snapshot with all metadata
+//! - `SharedMetadata` (shared.rs): Thread-safe wrapper with ArcSwap
+//! - Entity types: `StreamMeta`, `TopicMeta`, `PartitionMeta`, `UserMeta`, `ConsumerGroupMeta`
+//! - `SharedConsumerOffsetsStore`: Cross-shard consumer offsets visibility
+
+mod consumer_group;
+mod consumer_offsets_store;
+mod partition;
+mod shared;
+mod snapshot;
+mod stream;
+mod topic;
+mod user;
+
+pub use consumer_group::{ConsumerGroupMemberMeta, ConsumerGroupMeta};
+pub use consumer_offsets_store::SharedConsumerOffsetsStore;
+pub use partition::PartitionMeta;
+pub use shared::SharedMetadata;
+pub use snapshot::GlobalMetadata;
+pub use stream::StreamMeta;
+pub use topic::TopicMeta;
+pub use user::UserMeta;
+
+// Type aliases for clarity
+pub type StreamId = usize;
+pub type TopicId = usize;
+pub type PartitionId = usize;
+pub type UserId = u32;
+pub type ClientId = u32;
+pub type ConsumerGroupId = usize;
+pub type ConsumerGroupKey = (StreamId, TopicId, ConsumerGroupId);
diff --git a/core/server/src/metadata/partition.rs b/core/server/src/metadata/partition.rs
new file mode 100644
index 000000000..0b21fbc01
--- /dev/null
+++ b/core/server/src/metadata/partition.rs
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{PartitionId, StreamId, TopicId};
+use crate::streaming::stats::PartitionStats;
+use iggy_common::IggyTimestamp;
+use std::sync::Arc;
+
+#[derive(Clone, Debug)]
+pub struct PartitionMeta {
+ pub id: PartitionId,
+ pub stream_id: StreamId,
+ pub topic_id: TopicId,
+ pub created_at: IggyTimestamp,
+ /// Monotonically increasing version to detect stale partition_store entries.
+ /// Set to the GlobalMetadata version when the partition was created.
+ pub creation_version: u64,
+ pub stats: Arc<PartitionStats>,
+}
diff --git a/core/server/src/metadata/shared.rs b/core/server/src/metadata/shared.rs
new file mode 100644
index 000000000..d4c0be281
--- /dev/null
+++ b/core/server/src/metadata/shared.rs
@@ -0,0 +1,1385 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{
+ ConsumerGroupId, ConsumerGroupKey, ConsumerGroupMemberMeta, ConsumerGroupMeta, GlobalMetadata,
+ PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta,
+};
+use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
+use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
+use arc_swap::{ArcSwap, Guard};
+use iggy_common::sharding::IggyNamespace;
+use iggy_common::{
+ CompressionAlgorithm, IdKind, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize,
+};
+use imbl::HashSet as ImHashSet;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
+
+/// Thread-safe wrapper for GlobalMetadata using ArcSwap for lock-free reads.
+pub struct SharedMetadata {
+ inner: ArcSwap<GlobalMetadata>,
+ partition_counters: dashmap::DashMap<(StreamId, TopicId), AtomicUsize>,
+ member_partition_indices:
+ dashmap::DashMap<(StreamId, TopicId, ConsumerGroupId, usize), AtomicUsize>,
+ /// Atomic counters for ID generation - ensures unique IDs even under concurrent access
+ next_stream_id_counter: AtomicUsize,
+ next_user_id_counter: AtomicU32,
+ /// Per-stream topic ID counters
+ topic_id_counters: dashmap::DashMap<StreamId, AtomicUsize>,
+ /// Per-topic partition ID counters
+ partition_id_counters: dashmap::DashMap<(StreamId, TopicId), AtomicUsize>,
+ /// Per-topic consumer group ID counters
+ consumer_group_id_counters: dashmap::DashMap<(StreamId, TopicId), AtomicUsize>,
+}
+
+impl Default for SharedMetadata {
+ fn default() -> Self {
+ Self::new(GlobalMetadata::new())
+ }
+}
+
+impl SharedMetadata {
+ pub fn new(initial: GlobalMetadata) -> Self {
+ // Initialize counters based on existing data
+ let next_stream_id = initial
+ .streams
+ .keys()
+ .max()
+ .copied()
+ .map(|m| m + 1)
+ .unwrap_or(0);
+ let next_user_id = initial
+ .users
+ .keys()
+ .max()
+ .copied()
+ .map(|m| m + 1)
+ .unwrap_or(0);
+
+ Self {
+ inner: ArcSwap::from_pointee(initial),
+ partition_counters: dashmap::DashMap::new(),
+ member_partition_indices: dashmap::DashMap::new(),
+ next_stream_id_counter: AtomicUsize::new(next_stream_id),
+ next_user_id_counter: AtomicU32::new(next_user_id),
+ topic_id_counters: dashmap::DashMap::new(),
+ partition_id_counters: dashmap::DashMap::new(),
+ consumer_group_id_counters: dashmap::DashMap::new(),
+ }
+ }
+
+ #[inline]
+ pub fn load(&self) -> Guard<Arc<GlobalMetadata>> {
+ self.inner.load()
+ }
+
+ #[inline]
+ pub fn load_full(&self) -> Arc<GlobalMetadata> {
+ self.inner.load_full()
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Stream operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_stream(&self, id: StreamId, meta: StreamMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+ new.stream_index = new.stream_index.update(meta.name.clone(), id);
+ new.streams = new.streams.update(id, meta);
+ new.topics_by_stream = new.topics_by_stream.update(id, ImHashSet::new());
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn update_stream(&self, id: StreamId, name: Arc<str>) {
+ self.inner.rcu(move |current| {
+ let name = name.clone();
+ let mut new = (**current).clone();
+ if let Some(old_meta) = new.streams.get(&id) {
+ new.stream_index = new.stream_index.without(&old_meta.name);
+ let mut updated = old_meta.clone();
+ updated.name = name.clone();
+ new.streams = new.streams.update(id, updated);
+ new.stream_index = new.stream_index.update(name, id);
+ }
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_stream(&self, id: StreamId) {
+ let (topics_to_cleanup, groups_to_cleanup): (
+ Vec<TopicId>,
+ Vec<(TopicId, ConsumerGroupId)>,
+ ) = {
+ let current = self.inner.load();
+ let mut topics = Vec::new();
+ let mut groups = Vec::new();
+ if let Some(topic_ids) = current.topics_by_stream.get(&id) {
+ for &topic_id in topic_ids.iter() {
+ topics.push(topic_id);
+ let topic_key = (id, topic_id);
+ if let Some(group_ids) = current.consumer_groups_by_topic.get(&topic_key) {
+ for &group_id in group_ids.iter() {
+ groups.push((topic_id, group_id));
+ }
+ }
+ }
+ }
+ (topics, groups)
+ };
+
+ for topic_id in topics_to_cleanup {
+ self.partition_counters.remove(&(id, topic_id));
+ }
+
+ for (topic_id, group_id) in groups_to_cleanup {
+ self.remove_all_member_partition_indices(id, topic_id, group_id);
+ }
+
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+
+ if let Some(stream) = new.streams.get(&id) {
+ new.stream_index = new.stream_index.without(&stream.name);
+ }
+ new.streams = new.streams.without(&id);
+
+ if let Some(topic_ids) = new.topics_by_stream.get(&id) {
+ for &topic_id in topic_ids.iter() {
+ let topic_key = (id, topic_id);
+
+ if let Some(topic) = new.topics.get(&topic_key) {
+ new.topic_index = new.topic_index.without(&(id, topic.name.clone()));
+ }
+ new.topics = new.topics.without(&topic_key);
+
+ if let Some(partition_ids) = new.partitions_by_topic.get(&topic_key) {
+ for &partition_id in partition_ids.iter() {
+ let ns = IggyNamespace::new(id, topic_id, partition_id);
+ new.partitions = new.partitions.without(&ns);
+ }
+ }
+ new.partitions_by_topic = new.partitions_by_topic.without(&topic_key);
+
+ if let Some(group_ids) = new.consumer_groups_by_topic.get(&topic_key) {
+ for &group_id in group_ids.iter() {
+ let group_key = (id, topic_id, group_id);
+ if let Some(group) = new.consumer_groups.get(&group_key) {
+ new.consumer_group_index = new.consumer_group_index.without(&(
+ id,
+ topic_id,
+ group.name.clone(),
+ ));
+ }
+ new.consumer_groups = new.consumer_groups.without(&group_key);
+ }
+ }
+ new.consumer_groups_by_topic = new.consumer_groups_by_topic.without(&topic_key);
+ }
+ }
+ new.topics_by_stream = new.topics_by_stream.without(&id);
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Topic operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_topic(&self, stream_id: StreamId, id: TopicId, meta: TopicMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+ let topic_key = (stream_id, id);
+
+ new.topic_index = new.topic_index.update((stream_id, meta.name.clone()), id);
+ new.topics = new.topics.update(topic_key, meta);
+
+ let topics = new
+ .topics_by_stream
+ .get(&stream_id)
+ .cloned()
+ .unwrap_or_default();
+ new.topics_by_stream = new.topics_by_stream.update(stream_id, topics.update(id));
+
+ new.partitions_by_topic = new.partitions_by_topic.update(topic_key, ImHashSet::new());
+ new.consumer_groups_by_topic = new
+ .consumer_groups_by_topic
+ .update(topic_key, ImHashSet::new());
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn update_topic(&self, stream_id: StreamId, id: TopicId, meta: TopicMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+ let topic_key = (stream_id, id);
+
+ if let Some(old_meta) = new.topics.get(&topic_key)
+ && old_meta.name != meta.name
+ {
+ new.topic_index = new.topic_index.without(&(stream_id, old_meta.name.clone()));
+ new.topic_index = new.topic_index.update((stream_id, meta.name.clone()), id);
+ }
+
+ new.topics = new.topics.update(topic_key, meta);
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_topic(&self, stream_id: StreamId, topic_id: TopicId) {
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ let topic_key = (stream_id, topic_id);
+
+ if let Some(topic) = new.topics.get(&topic_key) {
+ new.topic_index = new.topic_index.without(&(stream_id, topic.name.clone()));
+ }
+ new.topics = new.topics.without(&topic_key);
+
+ if let Some(topics) = new.topics_by_stream.get(&stream_id) {
+ new.topics_by_stream = new
+ .topics_by_stream
+ .update(stream_id, topics.without(&topic_id));
+ }
+
+ if let Some(partition_ids) = new.partitions_by_topic.get(&topic_key) {
+ for &partition_id in partition_ids.iter() {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ new.partitions = new.partitions.without(&ns);
+ }
+ }
+ new.partitions_by_topic = new.partitions_by_topic.without(&topic_key);
+
+ if let Some(group_ids) = new.consumer_groups_by_topic.get(&topic_key) {
+ for &group_id in group_ids.iter() {
+ let group_key = (stream_id, topic_id, group_id);
+ if let Some(group) = new.consumer_groups.get(&group_key) {
+ new.consumer_group_index = new.consumer_group_index.without(&(
+ stream_id,
+ topic_id,
+ group.name.clone(),
+ ));
+ }
+ new.consumer_groups = new.consumer_groups.without(&group_key);
+ }
+ }
+ new.consumer_groups_by_topic = new.consumer_groups_by_topic.without(&topic_key);
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Partition operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_partitions(&self, partitions: Vec<PartitionMeta>) {
+ if partitions.is_empty() {
+ return;
+ }
+ self.inner.rcu(move |current| {
+ let partitions = partitions.clone();
+ let mut new = (**current).clone();
+ let new_version = new.version + 1;
+
+ for meta in partitions {
+ let ns = IggyNamespace::new(meta.stream_id, meta.topic_id, meta.id);
+ let topic_key = (meta.stream_id, meta.topic_id);
+
+ let meta_with_version = PartitionMeta {
+ creation_version: new_version,
+ ..meta
+ };
+ new.partitions = new.partitions.update(ns, meta_with_version);
+
+ let parts = new
+ .partitions_by_topic
+ .get(&topic_key)
+ .cloned()
+ .unwrap_or_default();
+ new.partitions_by_topic = new
+ .partitions_by_topic
+ .update(topic_key, parts.update(meta.id));
+ }
+
+ new.version = new_version;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_partitions(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_ids: &[PartitionId],
+ ) {
+ if partition_ids.is_empty() {
+ return;
+ }
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ let topic_key = (stream_id, topic_id);
+
+ for &partition_id in partition_ids {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ new.partitions = new.partitions.without(&ns);
+
+ if let Some(parts) = new.partitions_by_topic.get(&topic_key) {
+ new.partitions_by_topic = new
+ .partitions_by_topic
+ .update(topic_key, parts.without(&partition_id));
+ }
+ }
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // User operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_user(&self, id: UserId, meta: UserMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+ new.user_index = new.user_index.update(meta.username.clone(), id);
+ new.users = new.users.update(id, meta);
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn update_user(&self, id: UserId, meta: UserMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+
+ if let Some(old_meta) = new.users.get(&id)
+ && old_meta.username != meta.username
+ {
+ new.user_index = new.user_index.without(&old_meta.username);
+ new.user_index = new.user_index.update(meta.username.clone(), id);
+ }
+
+ new.users = new.users.update(id, meta);
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_user(&self, id: UserId) {
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ if let Some(user) = new.users.get(&id) {
+ new.user_index = new.user_index.without(&user.username);
+ }
+ new.users = new.users.without(&id);
+ new.personal_access_tokens = new.personal_access_tokens.without(&id);
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Personal access token operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_personal_access_token(&self, user_id: UserId, pat: PersonalAccessToken) {
+ self.inner.rcu(move |current| {
+ let pat = pat.clone();
+ let mut new = (**current).clone();
+ let user_pats = new
+ .personal_access_tokens
+ .get(&user_id)
+ .cloned()
+ .unwrap_or_default();
+ new.personal_access_tokens = new
+ .personal_access_tokens
+ .update(user_id, user_pats.update(pat.token.clone(), pat));
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_personal_access_token(&self, user_id: UserId, token_hash: &Arc<String>) {
+ let token_hash = token_hash.clone();
+ self.inner.rcu(move |current| {
+ let mut new = (**current).clone();
+ if let Some(user_pats) = new.personal_access_tokens.get(&user_id) {
+ new.personal_access_tokens = new
+ .personal_access_tokens
+ .update(user_id, user_pats.without(&token_hash));
+ }
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn get_user_personal_access_tokens(&self, user_id: UserId) -> Vec<PersonalAccessToken> {
+ self.load()
+ .personal_access_tokens
+ .get(&user_id)
+ .map(|pats| pats.values().cloned().collect())
+ .unwrap_or_default()
+ }
+
+ pub fn get_personal_access_token_by_hash(
+ &self,
+ token_hash: &str,
+ ) -> Option<PersonalAccessToken> {
+ let token_hash_arc = Arc::new(token_hash.to_string());
+ let metadata = self.load();
+ for (_, user_pats) in metadata.personal_access_tokens.iter() {
+ if let Some(pat) = user_pats.get(&token_hash_arc) {
+ return Some(pat.clone());
+ }
+ }
+ None
+ }
+
+ pub fn user_pat_count(&self, user_id: UserId) -> usize {
+ self.load()
+ .personal_access_tokens
+ .get(&user_id)
+ .map(|pats| pats.len())
+ .unwrap_or(0)
+ }
+
+ pub fn user_has_pat_with_name(&self, user_id: UserId, name: &str) -> bool {
+ self.load()
+ .personal_access_tokens
+ .get(&user_id)
+ .map(|pats| pats.values().any(|pat| pat.name.as_str() == name))
+ .unwrap_or(false)
+ }
+
+ pub fn find_pat_token_hash_by_name(&self, user_id: UserId, name: &str) -> Option<Arc<String>> {
+ self.load()
+ .personal_access_tokens
+ .get(&user_id)
+ .and_then(|pats| {
+ pats.iter()
+ .find(|(_, pat)| pat.name.as_str() == name)
+ .map(|(hash, _)| hash.clone())
+ })
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Consumer group operations
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn add_consumer_group(&self, key: ConsumerGroupKey, meta: ConsumerGroupMeta) {
+ self.inner.rcu(move |current| {
+ let meta = meta.clone();
+ let mut new = (**current).clone();
+ let (stream_id, topic_id, group_id) = key;
+ let topic_key = (stream_id, topic_id);
+
+ new.consumer_group_index = new
+ .consumer_group_index
+ .update((stream_id, topic_id, meta.name.clone()), group_id);
+ new.consumer_groups = new.consumer_groups.update(key, meta);
+
+ let groups = new
+ .consumer_groups_by_topic
+ .get(&topic_key)
+ .cloned()
+ .unwrap_or_default();
+ new.consumer_groups_by_topic = new
+ .consumer_groups_by_topic
+ .update(topic_key, groups.update(group_id));
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn delete_consumer_group(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ ) {
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ let key = (stream_id, topic_id, group_id);
+ let topic_key = (stream_id, topic_id);
+
+ if let Some(group) = new.consumer_groups.get(&key) {
+ new.consumer_group_index =
+ new.consumer_group_index
+ .without(&(stream_id, topic_id, group.name.clone()));
+ }
+ new.consumer_groups = new.consumer_groups.without(&key);
+
+ if let Some(groups) = new.consumer_groups_by_topic.get(&topic_key) {
+ new.consumer_groups_by_topic = new
+ .consumer_groups_by_topic
+ .update(topic_key, groups.without(&group_id));
+ }
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ pub fn join_consumer_group(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ client_id: u32,
+ ) -> Option<usize> {
+ let mut member_id = None;
+
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ let key = (stream_id, topic_id, group_id);
+
+ if let Some(group) = new.consumer_groups.get(&key) {
+ let mut updated_group = group.clone();
+
+ let next_id = updated_group
+ .members
+ .iter()
+ .map(|m| m.id)
+ .max()
+ .map(|m| m + 1)
+ .unwrap_or(0);
+
+ let new_member = ConsumerGroupMemberMeta::new(next_id, client_id);
+ updated_group.members.push_back(new_member);
+ updated_group.rebalance_members();
+
+ member_id = Some(next_id);
+ new.consumer_groups = new.consumer_groups.update(key, updated_group);
+ }
+
+ new.version += 1;
+ Arc::new(new)
+ });
+
+ member_id
+ }
+
+ pub fn leave_consumer_group(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ client_id: u32,
+ ) -> Option<usize> {
+ let mut member_id = None;
+
+ self.inner.rcu(|current| {
+ let mut new = (**current).clone();
+ let key = (stream_id, topic_id, group_id);
+
+ if let Some(group) = new.consumer_groups.get(&key) {
+ let mut updated_group = group.clone();
+
+ if let Some(idx) = updated_group
+ .members
+ .iter()
+ .position(|m| m.client_id == client_id)
+ {
+ member_id = Some(updated_group.members[idx].id);
+ updated_group.members.remove(idx);
+ updated_group.rebalance_members();
+ new.consumer_groups = new.consumer_groups.update(key, updated_group);
+ }
+ }
+
+ new.version += 1;
+ Arc::new(new)
+ });
+
+ // Clean up member_partition_indices entry for the leaving member
+ if let Some(id) = member_id {
+ self.remove_member_partition_index(stream_id, topic_id, group_id, id);
+ }
+
+ member_id
+ }
+
+ pub fn is_consumer_group_member(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ client_id: u32,
+ ) -> bool {
+ let metadata = self.load();
+ let key = (stream_id, topic_id, group_id);
+ metadata
+ .consumer_groups
+ .get(&key)
+ .map(|g| g.members.iter().any(|m| m.client_id == client_id))
+ .unwrap_or(false)
+ }
+
+ pub fn rebalance_consumer_groups_for_topic(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partition_ids: &[PartitionId],
+ ) {
+ self.inner.rcu(move |current| {
+ let mut new = (**current).clone();
+
+ let cg_ids: Vec<_> = new
+ .consumer_groups_by_topic
+ .get(&(stream_id, topic_id))
+ .map(|set| set.iter().copied().collect())
+ .unwrap_or_default();
+
+ for cg_id in cg_ids {
+ let key = (stream_id, topic_id, cg_id);
+ if let Some(mut group) = new.consumer_groups.get(&key).cloned() {
+ group.partitions = partition_ids.to_vec();
+ group.rebalance_members();
+ new.consumer_groups = new.consumer_groups.update(key, group);
+ }
+ }
+
+ new.version += 1;
+ Arc::new(new)
+ });
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Lookup helpers
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn get_stream_id(&self, identifier: &Identifier) -> Option<StreamId> {
+ let metadata = self.load();
+ match identifier.kind {
+ IdKind::Numeric => {
+ let stream_id = identifier.get_u32_value().ok()? as StreamId;
+ // Validate that stream actually exists
+ if metadata.streams.contains_key(&stream_id) {
+ Some(stream_id)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = identifier.get_cow_str_value().ok()?;
+ metadata.stream_index.get(name.as_ref()).copied()
+ }
+ }
+ }
+
+ pub fn stream_name_exists(&self, name: &str) -> bool {
+ self.load().stream_index.contains_key(name)
+ }
+
+ /// Returns the next stream ID using atomic counter.
+ /// Note: For new code, prefer `try_register_stream` which is race-free.
+ pub fn next_stream_id(&self) -> StreamId {
+ self.next_stream_id_counter.fetch_add(1, Ordering::AcqRel)
+ }
+
+ /// Returns the next topic ID for a stream using atomic counter.
+ /// Note: For new code, prefer `try_register_topic` which is race-free.
+ pub fn next_topic_id(&self, stream_id: StreamId) -> TopicId {
+ self.topic_id_counters
+ .entry(stream_id)
+ .or_insert_with(|| {
+ // Initialize counter based on existing topics
+ let max_id = self
+ .load()
+ .topics_by_stream
+ .get(&stream_id)
+ .and_then(|topics| topics.iter().max().copied())
+ .unwrap_or(0);
+ AtomicUsize::new(if max_id == 0 { 0 } else { max_id + 1 })
+ })
+ .fetch_add(1, Ordering::AcqRel)
+ }
+
+ /// Returns the next partition ID for a topic using atomic counter.
+ pub fn next_partition_id(&self, stream_id: StreamId, topic_id: TopicId) -> PartitionId {
+ self.partition_id_counters
+ .entry((stream_id, topic_id))
+ .or_insert_with(|| {
+ // Initialize counter based on existing partitions
+ let max_id = self
+ .load()
+ .partitions_by_topic
+ .get(&(stream_id, topic_id))
+ .and_then(|partitions| partitions.iter().max().copied())
+ .unwrap_or(0);
+ AtomicUsize::new(if max_id == 0 { 0 } else { max_id + 1 })
+ })
+ .fetch_add(1, Ordering::AcqRel)
+ }
+
+ pub fn get_topic_id(&self, stream_id: StreamId, identifier: &Identifier) -> Option<TopicId> {
+ let metadata = self.load();
+ match identifier.kind {
+ IdKind::Numeric => {
+ let topic_id = identifier.get_u32_value().ok()? as TopicId;
+ // Validate that topic actually exists
+ if metadata.topics.contains_key(&(stream_id, topic_id)) {
+ Some(topic_id)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = identifier.get_cow_str_value().ok()?;
+ metadata
+ .topic_index
+ .get(&(stream_id, Arc::from(name.as_ref())))
+ .copied()
+ }
+ }
+ }
+
+ pub fn get_user_id(&self, identifier: &Identifier) -> Option<UserId> {
+ let metadata = self.load();
+ match identifier.kind {
+ IdKind::Numeric => Some(identifier.get_u32_value().ok()?),
+ IdKind::String => {
+ let name = identifier.get_cow_str_value().ok()?;
+ metadata.user_index.get(name.as_ref()).copied()
+ }
+ }
+ }
+
+ pub fn get_consumer_group_id(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ identifier: &Identifier,
+ ) -> Option<ConsumerGroupId> {
+ let metadata = self.load();
+ match identifier.kind {
+ IdKind::Numeric => {
+ let group_id = identifier.get_u32_value().ok()? as ConsumerGroupId;
+ // Validate that consumer group actually exists
+ if metadata
+ .consumer_groups
+ .contains_key(&(stream_id, topic_id, group_id))
+ {
+ Some(group_id)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = identifier.get_cow_str_value().ok()?;
+ metadata
+ .consumer_group_index
+ .get(&(stream_id, topic_id, Arc::from(name.as_ref())))
+ .copied()
+ }
+ }
+ }
+
+ pub fn stream_exists(&self, id: StreamId) -> bool {
+ self.load().streams.contains_key(&id)
+ }
+
+ pub fn topic_exists(&self, stream_id: StreamId, topic_id: TopicId) -> bool {
+ self.load().topics.contains_key(&(stream_id, topic_id))
+ }
+
+ pub fn partition_exists(&self, ns: &IggyNamespace) -> bool {
+ self.load().partitions.contains_key(ns)
+ }
+
+ pub fn user_exists(&self, id: UserId) -> bool {
+ self.load().users.contains_key(&id)
+ }
+
+ pub fn consumer_group_exists(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ ) -> bool {
+ self.load()
+ .consumer_groups
+ .contains_key(&(stream_id, topic_id, group_id))
+ }
+
+ pub fn consumer_group_exists_by_name(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ name: &str,
+ ) -> bool {
+ self.load()
+ .consumer_group_index
+ .contains_key(&(stream_id, topic_id, Arc::from(name)))
+ }
+
+ /// Returns the next consumer group ID for a topic using atomic counter.
+ pub fn next_consumer_group_id(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ ) -> ConsumerGroupId {
+ self.consumer_group_id_counters
+ .entry((stream_id, topic_id))
+ .or_insert_with(|| {
+ // Initialize counter based on existing consumer groups
+ let max_id = self
+ .load()
+ .consumer_groups_by_topic
+ .get(&(stream_id, topic_id))
+ .and_then(|groups| groups.iter().max().copied())
+ .unwrap_or(0);
+ AtomicUsize::new(if max_id == 0 { 0 } else { max_id + 1 })
+ })
+ .fetch_add(1, Ordering::AcqRel)
+ }
+
+ pub fn streams_count(&self) -> usize {
+ self.load().streams.len()
+ }
+
+ pub fn topics_count(&self, stream_id: StreamId) -> usize {
+ self.load()
+ .topics_by_stream
+ .get(&stream_id)
+ .map(|t| t.len())
+ .unwrap_or(0)
+ }
+
+ pub fn partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize {
+ self.load()
+ .partitions_by_topic
+ .get(&(stream_id, topic_id))
+ .map(|p| p.len())
+ .unwrap_or(0)
+ }
+
+ pub fn get_next_partition_id(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ partitions_count: usize,
+ ) -> usize {
+ if partitions_count == 0 {
+ return 0;
+ }
+
+ let key = (stream_id, topic_id);
+ let counter = self
+ .partition_counters
+ .entry(key)
+ .or_insert_with(|| std::sync::atomic::AtomicUsize::new(0));
+
+ let mut partition_id = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+ if partition_id >= partitions_count {
+ partition_id %= partitions_count;
+ counter.store(partition_id + 1, std::sync::atomic::Ordering::Relaxed);
+ }
+ partition_id
+ }
+
+ pub fn get_next_member_partition_id(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ member_id: usize,
+ assigned_partitions: &[PartitionId],
+ calculate: bool,
+ ) -> Option<PartitionId> {
+ if assigned_partitions.is_empty() {
+ return None;
+ }
+
+ let key = (stream_id, topic_id, group_id, member_id);
+ let counter = self
+ .member_partition_indices
+ .entry(key)
+ .or_insert_with(|| std::sync::atomic::AtomicUsize::new(0));
+
+ let partitions_count = assigned_partitions.len();
+
+ if calculate {
+ let current = counter
+ .fetch_update(
+ std::sync::atomic::Ordering::Relaxed,
+ std::sync::atomic::Ordering::Relaxed,
+ |current| Some((current + 1) % partitions_count),
+ )
+ .unwrap();
+ Some(assigned_partitions[current])
+ } else {
+ let current = counter.load(std::sync::atomic::Ordering::Relaxed);
+ Some(assigned_partitions[current % partitions_count])
+ }
+ }
+
+ pub fn remove_member_partition_index(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ member_id: usize,
+ ) {
+ let key = (stream_id, topic_id, group_id, member_id);
+ self.member_partition_indices.remove(&key);
+ }
+
+ pub fn remove_all_member_partition_indices(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: ConsumerGroupId,
+ ) {
+ let member_ids: Vec<usize> = {
+ let metadata = self.load();
+ let key = (stream_id, topic_id, group_id);
+ metadata
+ .consumer_groups
+ .get(&key)
+ .map(|cg| cg.members.iter().map(|m| m.id).collect())
+ .unwrap_or_default()
+ };
+
+ for member_id in member_ids {
+ let key = (stream_id, topic_id, group_id, member_id);
+ self.member_partition_indices.remove(&key);
+ }
+ }
+
+ pub fn users_count(&self) -> usize {
+ self.load().users.len()
+ }
+
+ pub fn username_exists(&self, username: &str) -> bool {
+ self.load().user_index.contains_key(username)
+ }
+
+ /// Returns the next user ID using atomic counter.
+ pub fn next_user_id(&self) -> UserId {
+ self.next_user_id_counter.fetch_add(1, Ordering::AcqRel)
+ }
+
+ pub fn get_user(&self, id: UserId) -> Option<UserMeta> {
+ self.load().users.get(&id).cloned()
+ }
+
+ pub fn get_all_users(&self) -> Vec<UserMeta> {
+ self.load().users.values().cloned().collect()
+ }
+
+ pub fn consumer_groups_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize {
+ self.load()
+ .consumer_groups_by_topic
+ .get(&(stream_id, topic_id))
+ .map(|g| g.len())
+ .unwrap_or(0)
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Stats getters
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn get_stream_stats(&self, id: StreamId) -> Option<Arc<StreamStats>> {
+ self.load().streams.get(&id).map(|s| s.stats.clone())
+ }
+
+ pub fn get_topic_stats(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ ) -> Option<Arc<TopicStats>> {
+ self.load()
+ .topics
+ .get(&(stream_id, topic_id))
+ .map(|t| t.stats.clone())
+ }
+
+ pub fn get_partition_stats(&self, ns: &IggyNamespace) -> Option<Arc<PartitionStats>> {
+ self.load().partitions.get(ns).map(|p| p.stats.clone())
+ }
+
+ // ═══════════════════════════════════════════════════════════════
+ // Stats registration
+ // ═══════════════════════════════════════════════════════════════
+
+ pub fn register_stream(
+ &self,
+ id: StreamId,
+ name: Arc<str>,
+ created_at: IggyTimestamp,
+ ) -> Arc<StreamStats> {
+ let stats = Arc::new(StreamStats::default());
+ let meta = StreamMeta::with_stats(id, name, created_at, stats.clone());
+ self.add_stream(id, meta);
+ stats
+ }
+
+ /// Atomically validates name uniqueness, allocates ID, and registers stream.
+ /// Prevents TOCTOU races by doing all operations in a single RCU transaction.
+ pub fn try_register_stream(
+ &self,
+ name: Arc<str>,
+ created_at: IggyTimestamp,
+ ) -> Result<(StreamId, Arc<StreamStats>), IggyError> {
+ // Atomically allocate a unique ID (even if registration fails, ID is "used")
+ let stream_id = self.next_stream_id_counter.fetch_add(1, Ordering::AcqRel);
+ let stats = Arc::new(StreamStats::default());
+
+ // Use atomic flags to capture result from inside RCU closure
+ let name_existed = Arc::new(AtomicBool::new(false));
+ let name_existed_clone = name_existed.clone();
+ let name_clone = name.clone();
+ let stats_clone = stats.clone();
+
+ self.inner.rcu(move |current| {
+ // Check if name already exists
+ if current.stream_index.contains_key(&name_clone) {
+ name_existed_clone.store(true, Ordering::Release);
+ return Arc::clone(current); // No modification
+ }
+
+ name_existed_clone.store(false, Ordering::Release);
+
+ let meta = StreamMeta::with_stats(
+ stream_id,
+ name_clone.clone(),
+ created_at,
+ stats_clone.clone(),
+ );
+ let mut new = (**current).clone();
+ new.stream_index = new.stream_index.update(name_clone.clone(), stream_id);
+ new.streams = new.streams.update(stream_id, meta);
+ new.topics_by_stream = new.topics_by_stream.update(stream_id, ImHashSet::new());
+ new.version += 1;
+ Arc::new(new)
+ });
+
+ if name_existed.load(Ordering::Acquire) {
+ Err(IggyError::StreamNameAlreadyExists(name.to_string()))
+ } else {
+ Ok((stream_id, stats))
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ pub fn register_topic(
+ &self,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ name: Arc<str>,
+ created_at: IggyTimestamp,
+ message_expiry: IggyExpiry,
+ compression_algorithm: CompressionAlgorithm,
+ max_topic_size: MaxTopicSize,
+ replication_factor: u8,
+ partitions_count: u32,
+ ) -> Arc<TopicStats> {
+ let parent_stats = self
+ .get_stream_stats(stream_id)
+ .expect("Parent stream stats must exist before registering topic");
+
+ let stats = Arc::new(TopicStats::new(parent_stats));
+ let meta = TopicMeta {
+ id: topic_id,
+ stream_id,
+ name,
+ created_at,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ partitions_count,
+ stats: stats.clone(),
+ };
+ self.add_topic(stream_id, topic_id, meta);
+ stats
+ }
+
+ /// Atomically validates name uniqueness, allocates ID, and registers topic.
+ /// Prevents TOCTOU races by doing all operations in a single RCU transaction.
+ #[allow(clippy::too_many_arguments)]
+ pub fn try_register_topic(
+ &self,
+ stream_id: StreamId,
+ name: Arc<str>,
+ created_at: IggyTimestamp,
+ message_expiry: IggyExpiry,
+ compression_algorithm: CompressionAlgorithm,
+ max_topic_size: MaxTopicSize,
+ replication_factor: u8,
+ partitions_count: u32,
+ ) -> Result<(TopicId, Arc<TopicStats>), IggyError> {
+ // Get parent stream stats (required for topic stats hierarchy)
+ let parent_stats = self.get_stream_stats(stream_id).ok_or_else(|| {
+ IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap())
+ })?;
+
+ // Atomically allocate a unique topic ID for this stream
+ let topic_id = self
+ .topic_id_counters
+ .entry(stream_id)
+ .or_insert_with(|| {
+ // Initialize counter based on existing topics
+ let max_id = self
+ .load()
+ .topics_by_stream
+ .get(&stream_id)
+ .and_then(|topics| topics.iter().max().copied())
+ .unwrap_or(0);
+ AtomicUsize::new(if max_id == 0 { 0 } else { max_id + 1 })
+ })
+ .fetch_add(1, Ordering::AcqRel);
+
+ let stats = Arc::new(TopicStats::new(parent_stats));
+
+ // Use atomic flags to capture result from inside RCU closure
+ let name_existed = Arc::new(AtomicBool::new(false));
+ let name_existed_clone = name_existed.clone();
+ let name_clone = name.clone();
+ let stats_clone = stats.clone();
+
+ self.inner.rcu(move |current| {
+ // Check if topic name already exists in this stream
+ let topic_name_key = (stream_id, name_clone.clone());
+ if current.topic_index.contains_key(&topic_name_key) {
+ name_existed_clone.store(true, Ordering::Release);
+ return Arc::clone(current); // No modification
+ }
+
+ name_existed_clone.store(false, Ordering::Release);
+
+ let meta = TopicMeta {
+ id: topic_id,
+ stream_id,
+ name: name_clone.clone(),
+ created_at,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ partitions_count,
+ stats: stats_clone.clone(),
+ };
+
+ let mut new = (**current).clone();
+ new.topic_index = new
+ .topic_index
+ .update((stream_id, name_clone.clone()), topic_id);
+ new.topics = new.topics.update((stream_id, topic_id), meta);
+
+ let topics = new
+ .topics_by_stream
+ .get(&stream_id)
+ .cloned()
+ .unwrap_or_default();
+ new.topics_by_stream = new
+ .topics_by_stream
+ .update(stream_id, topics.update(topic_id));
+ new.partitions_by_topic = new
+ .partitions_by_topic
+ .update((stream_id, topic_id), ImHashSet::new());
+ new.consumer_groups_by_topic = new
+ .consumer_groups_by_topic
+ .update((stream_id, topic_id), ImHashSet::new());
+ new.version += 1;
+ Arc::new(new)
+ });
+
+ if name_existed.load(Ordering::Acquire) {
+ Err(IggyError::TopicNameAlreadyExists(
+ name.to_string(),
+ Identifier::numeric(stream_id as u32).unwrap(),
+ ))
+ } else {
+ Ok((topic_id, stats))
+ }
+ }
+
+ pub fn register_partition(
+ &self,
+ ns: IggyNamespace,
+ created_at: IggyTimestamp,
+ ) -> Arc<PartitionStats> {
+ let parent_stats = self
+ .get_topic_stats(ns.stream_id(), ns.topic_id())
+ .expect("Parent topic stats must exist before registering partition");
+
+ let stats = Arc::new(PartitionStats::new(parent_stats));
+ let meta = PartitionMeta {
+ id: ns.partition_id(),
+ stream_id: ns.stream_id(),
+ topic_id: ns.topic_id(),
+ created_at,
+ creation_version: 0,
+ stats: stats.clone(),
+ };
+ self.add_partitions(vec![meta]);
+ stats
+ }
+
+ pub fn register_partitions(
+ &self,
+ namespaces: &[IggyNamespace],
+ created_at: IggyTimestamp,
+ ) -> Vec<Arc<PartitionStats>> {
+ if namespaces.is_empty() {
+ return Vec::new();
+ }
+
+ let first = &namespaces[0];
+ let parent_stats = self
+ .get_topic_stats(first.stream_id(), first.topic_id())
+ .expect("Parent topic stats must exist before registering partitions");
+
+ let mut stats_list = Vec::with_capacity(namespaces.len());
+ let mut metas = Vec::with_capacity(namespaces.len());
+
+ for ns in namespaces {
+ let stats = Arc::new(PartitionStats::new(parent_stats.clone()));
+ metas.push(PartitionMeta {
+ id: ns.partition_id(),
+ stream_id: ns.stream_id(),
+ topic_id: ns.topic_id(),
+ created_at,
+ creation_version: 0,
+ stats: stats.clone(),
+ });
+ stats_list.push(stats);
+ }
+
+ self.add_partitions(metas);
+ stats_list
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_stream_crud() {
+ let metadata = SharedMetadata::default();
+
+ let stream_meta = StreamMeta::with_stats(
+ 1,
+ Arc::from("test-stream"),
+ IggyTimestamp::now(),
+ Arc::new(StreamStats::default()),
+ );
+ metadata.add_stream(1, stream_meta);
+
+ assert!(metadata.stream_exists(1));
+ assert_eq!(metadata.streams_count(), 1);
+
+ metadata.update_stream(1, Arc::from("renamed-stream"));
+ let loaded = metadata.load();
+ assert_eq!(
+ loaded.streams.get(&1).unwrap().name.as_ref(),
+ "renamed-stream"
+ );
+ assert!(loaded.stream_index.contains_key("renamed-stream"));
+ assert!(!loaded.stream_index.contains_key("test-stream"));
+
+ metadata.delete_stream(1);
+ assert!(!metadata.stream_exists(1));
+ assert_eq!(metadata.streams_count(), 0);
+ }
+
+ #[test]
+ fn test_cascade_delete() {
+ let metadata = SharedMetadata::default();
+
+ let stream_stats = Arc::new(StreamStats::default());
+ metadata.add_stream(
+ 1,
+ StreamMeta::with_stats(
+ 1,
+ Arc::from("stream-1"),
+ IggyTimestamp::now(),
+ stream_stats.clone(),
+ ),
+ );
+
+ let topic_stats = Arc::new(TopicStats::new(stream_stats));
+ metadata.add_topic(
+ 1,
+ 1,
+ TopicMeta {
+ id: 1,
+ stream_id: 1,
+ name: Arc::from("topic-1"),
+ created_at: IggyTimestamp::now(),
+ message_expiry: IggyExpiry::NeverExpire,
+ compression_algorithm: CompressionAlgorithm::None,
+ max_topic_size: MaxTopicSize::Unlimited,
+ replication_factor: 1,
+ partitions_count: 2,
+ stats: topic_stats.clone(),
+ },
+ );
+
+ metadata.add_partitions(vec![
+ PartitionMeta {
+ id: 0,
+ stream_id: 1,
+ topic_id: 1,
+ created_at: IggyTimestamp::now(),
+ creation_version: 0,
+ stats: Arc::new(PartitionStats::new(topic_stats.clone())),
+ },
+ PartitionMeta {
+ id: 1,
+ stream_id: 1,
+ topic_id: 1,
+ created_at: IggyTimestamp::now(),
+ creation_version: 0,
+ stats: Arc::new(PartitionStats::new(topic_stats)),
+ },
+ ]);
+
+ assert!(metadata.stream_exists(1));
+ assert!(metadata.topic_exists(1, 1));
+ assert!(metadata.partition_exists(&IggyNamespace::new(1, 1, 0)));
+ assert!(metadata.partition_exists(&IggyNamespace::new(1, 1, 1)));
+
+ metadata.delete_stream(1);
+
+ assert!(!metadata.stream_exists(1));
+ assert!(!metadata.topic_exists(1, 1));
+ assert!(!metadata.partition_exists(&IggyNamespace::new(1, 1, 0)));
+ assert!(!metadata.partition_exists(&IggyNamespace::new(1, 1, 1)));
+ }
+}
diff --git a/core/server/src/metadata/snapshot.rs b/core/server/src/metadata/snapshot.rs
new file mode 100644
index 000000000..84fb016dc
--- /dev/null
+++ b/core/server/src/metadata/snapshot.rs
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{
+ ConsumerGroupId, ConsumerGroupKey, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId,
+ StreamMeta, TopicId, TopicMeta, UserId, UserMeta,
+};
+use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
+use iggy_common::sharding::IggyNamespace;
+use imbl::{HashMap as ImHashMap, HashSet as ImHashSet};
+use std::sync::Arc;
+
+/// Immutable metadata snapshot using im::HashMap for O(log n) structural sharing.
+#[derive(Clone, Default)]
+pub struct GlobalMetadata {
+ pub streams: ImHashMap<StreamId, StreamMeta>,
+ pub topics: ImHashMap<(StreamId, TopicId), TopicMeta>,
+ pub partitions: ImHashMap<IggyNamespace, PartitionMeta>,
+ pub users: ImHashMap<UserId, UserMeta>,
+ pub consumer_groups: ImHashMap<ConsumerGroupKey, ConsumerGroupMeta>,
+
+ /// user_id -> (token_hash -> PAT)
+ pub personal_access_tokens: ImHashMap<UserId, ImHashMap<Arc<String>, PersonalAccessToken>>,
+
+ // Forward indexes (name → ID)
+ pub stream_index: ImHashMap<Arc<str>, StreamId>,
+ pub topic_index: ImHashMap<(StreamId, Arc<str>), TopicId>,
+ pub user_index: ImHashMap<Arc<str>, UserId>,
+ pub consumer_group_index: ImHashMap<(StreamId, TopicId, Arc<str>), ConsumerGroupId>,
+
+ // Reverse indexes (for efficient cascade delete)
+ pub topics_by_stream: ImHashMap<StreamId, ImHashSet<TopicId>>,
+ pub partitions_by_topic: ImHashMap<(StreamId, TopicId), ImHashSet<PartitionId>>,
+ pub consumer_groups_by_topic: ImHashMap<(StreamId, TopicId), ImHashSet<ConsumerGroupId>>,
+
+ /// Monotonic version for cache invalidation
+ pub version: u64,
+}
+
+impl GlobalMetadata {
+ pub fn new() -> Self {
+ Self::default()
+ }
+}
diff --git a/core/server/src/metadata/stream.rs b/core/server/src/metadata/stream.rs
new file mode 100644
index 000000000..a986bff13
--- /dev/null
+++ b/core/server/src/metadata/stream.rs
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::StreamId;
+use crate::streaming::stats::StreamStats;
+use iggy_common::IggyTimestamp;
+use std::sync::Arc;
+
+/// Stream metadata stored in the shared snapshot.
+#[derive(Clone, Debug)]
+pub struct StreamMeta {
+ pub id: StreamId,
+ pub name: Arc<str>,
+ pub created_at: IggyTimestamp,
+ pub stats: Arc<StreamStats>,
+}
+
+impl StreamMeta {
+ pub fn new(id: StreamId, name: Arc<str>, created_at: IggyTimestamp) -> Self {
+ Self {
+ id,
+ name,
+ created_at,
+ stats: Arc::new(StreamStats::default()),
+ }
+ }
+
+ pub fn with_stats(
+ id: StreamId,
+ name: Arc<str>,
+ created_at: IggyTimestamp,
+ stats: Arc<StreamStats>,
+ ) -> Self {
+ Self {
+ id,
+ name,
+ created_at,
+ stats,
+ }
+ }
+}
diff --git a/core/server/src/metadata/topic.rs b/core/server/src/metadata/topic.rs
new file mode 100644
index 000000000..ac4f856aa
--- /dev/null
+++ b/core/server/src/metadata/topic.rs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::{StreamId, TopicId};
+use crate::streaming::stats::TopicStats;
+use iggy_common::{CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize};
+use std::sync::Arc;
+
+/// Topic metadata stored in the shared snapshot.
+#[derive(Clone, Debug)]
+pub struct TopicMeta {
+ pub id: TopicId,
+ pub stream_id: StreamId,
+ pub name: Arc<str>,
+ pub created_at: IggyTimestamp,
+ pub message_expiry: IggyExpiry,
+ pub compression_algorithm: CompressionAlgorithm,
+ pub max_topic_size: MaxTopicSize,
+ pub replication_factor: u8,
+ pub partitions_count: u32,
+ pub stats: Arc<TopicStats>,
+}
diff --git a/core/server/src/metadata/user.rs b/core/server/src/metadata/user.rs
new file mode 100644
index 000000000..c82d49a44
--- /dev/null
+++ b/core/server/src/metadata/user.rs
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::metadata::UserId;
+use iggy_common::{IggyTimestamp, Permissions, UserStatus};
+use std::sync::Arc;
+
+#[derive(Clone, Debug)]
+pub struct UserMeta {
+ pub id: UserId,
+ pub username: Arc<str>,
+ pub password_hash: Arc<str>,
+ pub status: UserStatus,
+ pub permissions: Option<Arc<Permissions>>,
+ pub created_at: IggyTimestamp,
+}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 135a9b4bc..f23c8aca8 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -17,31 +17,33 @@
*/
use super::{
- IggyShard, TaskRegistry, transmission::connector::ShardConnector,
- transmission::frame::ShardFrame,
+ IggyShard, TaskRegistry, partition_store::PartitionDataStore,
+ transmission::connector::ShardConnector, transmission::frame::ShardFrame,
};
+use crate::metadata::{SharedConsumerOffsetsStore, SharedMetadata};
use crate::{
configs::server::ServerConfig,
- slab::{streams::Streams, users::Users},
state::file::FileState,
streaming::{
clients::client_manager::ClientManager, diagnostics::metrics::Metrics,
- utils::ptr::EternalPtr,
+ users::permissioner::Permissioner, utils::ptr::EternalPtr,
},
versioning::SemanticVersion,
};
use dashmap::DashMap;
use iggy_common::EncryptorKind;
use iggy_common::sharding::{IggyNamespace, PartitionLocation};
-use std::{cell::Cell, rc::Rc, sync::atomic::AtomicBool};
+use std::{
+ cell::{Cell, RefCell},
+ rc::Rc,
+ sync::atomic::AtomicBool,
+};
#[derive(Default)]
pub struct IggyShardBuilder {
id: Option<u16>,
- streams: Option<Streams>,
shards_table: Option<EternalPtr<DashMap<IggyNamespace, PartitionLocation>>>,
state: Option<FileState>,
- users: Option<Users>,
client_manager: Option<ClientManager>,
connections: Option<Vec<ShardConnector<ShardFrame>>>,
config: Option<ServerConfig>,
@@ -49,6 +51,8 @@ pub struct IggyShardBuilder {
version: Option<SemanticVersion>,
metrics: Option<Metrics>,
is_follower: bool,
+ shared_metadata: Option<&'static SharedMetadata>,
+ shared_consumer_offsets: Option<&'static SharedConsumerOffsetsStore>,
}
impl IggyShardBuilder {
@@ -90,21 +94,11 @@ impl IggyShardBuilder {
self
}
- pub fn streams(mut self, streams: Streams) -> Self {
- self.streams = Some(streams);
- self
- }
-
pub fn state(mut self, state: FileState) -> Self {
self.state = Some(state);
self
}
- pub fn users(mut self, users: Users) -> Self {
- self.users = Some(users);
- self
- }
-
pub fn metrics(mut self, metrics: Metrics) -> Self {
self.metrics = Some(metrics);
self
@@ -115,18 +109,33 @@ impl IggyShardBuilder {
self
}
+ pub fn shared_metadata(mut self, shared_metadata: &'static SharedMetadata) -> Self {
+ self.shared_metadata = Some(shared_metadata);
+ self
+ }
+
+ pub fn shared_consumer_offsets(
+ mut self,
+ shared_consumer_offsets: &'static SharedConsumerOffsetsStore,
+ ) -> Self {
+ self.shared_consumer_offsets = Some(shared_consumer_offsets);
+ self
+ }
+
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
pub fn build(self) -> IggyShard {
let id = self.id.unwrap();
- let streams = self.streams.unwrap();
let shards_table = self.shards_table.unwrap();
let state = self.state.unwrap();
- let users = self.users.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
let encryptor = self.encryptor;
let client_manager = self.client_manager.unwrap();
let version = self.version.unwrap();
+ let shared_metadata = self.shared_metadata.expect("shared_metadata is required");
+ let shared_consumer_offsets = self
+ .shared_consumer_offsets
+ .expect("shared_consumer_offsets is required");
let (stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
@@ -150,12 +159,17 @@ impl IggyShardBuilder {
// Trigger initial check in case servers bind before task starts
let _ = config_writer_notify.try_send(());
+ // Create per-shard stores (wrapped in RefCell for interior mutability)
+ let partition_store = RefCell::new(PartitionDataStore::new());
+
IggyShard {
id,
shards,
shards_table,
- streams, // TODO: Fixme
- users,
+ shared_metadata,
+ shared_consumer_offsets,
+ partition_store,
+ pending_partition_inits: RefCell::new(std::collections::HashSet::new()),
fs_locks: Default::default(),
encryptor,
config,
@@ -173,7 +187,7 @@ impl IggyShardBuilder {
config_writer_notify,
config_writer_receiver,
task_registry,
- permissioner: Default::default(),
+ permissioner: Permissioner::new(shared_metadata),
client_manager,
}
}
diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs
index 087b7a54d..148dd2fef 100644
--- a/core/server/src/shard/handlers.rs
+++ b/core/server/src/shard/handlers.rs
@@ -19,23 +19,24 @@ use super::*;
use crate::{
shard::{
IggyShard,
- namespace::IggyFullNamespace,
+ namespace::IggyNamespace,
transmission::{
event::ShardEvent,
frame::ShardResponse,
message::{ShardMessage, ShardRequest, ShardRequestPayload},
},
},
- streaming::{session::Session, traits::MainOps},
+ streaming::session::Session,
tcp::{
connection_handler::{ConnectionAction, handle_connection, handle_error},
tcp_listener::cleanup_connection,
},
};
use compio_net::TcpStream;
-use iggy_common::{Identifier, IggyError, SenderKind, TransportProtocol};
+use iggy_common::{Identifier, IggyError, IggyTimestamp, SenderKind, TransportProtocol};
use nix::sys::stat::SFlag;
use std::os::fd::{FromRawFd, IntoRawFd};
+use std::sync::Arc;
use tracing::info;
pub(super) async fn handle_shard_message(
@@ -63,35 +64,129 @@ async fn handle_request(
let partition_id = request.partition_id;
match request.payload {
ShardRequestPayload::SendMessages { batch } => {
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
let batch = shard.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
+
+ // Get numeric IDs for partition_store lookup
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ // Use partition_store - check for stale entries and initialize on-demand
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ // Get partition metadata from SharedMetadata (source of truth)
+ let metadata = shard.shared_metadata.load();
+ let partition_meta = metadata.partitions.get(&namespace);
+
+ // Check if partition_store entry exists AND is valid (not stale)
+ let needs_init = {
+ let store = shard.partition_store.borrow();
+ match (store.get(&namespace), partition_meta) {
+ (Some(data), Some(meta)) if data.creation_version == meta.creation_version => {
+ false
+ }
+ (Some(_), _) => {
+ // Entry exists but is stale (different version or no metadata)
+ drop(store);
+ shard.partition_store.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => true,
+ }
+ };
+
+ if needs_init {
+ // Partition not yet initialized or stale - initialize on-demand
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(IggyTimestamp::now);
+
+ shard
+ .init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ created_at,
+ )
+ .await?;
+ }
+
shard
- .streams
- .append_messages(&shard.config.system, &shard.task_registry, &ns, batch)
+ .append_messages_to_partition_store(&namespace, batch, &shard.config.system)
.await?;
+
shard.metrics.increment_messages(messages_count as u64);
Ok(ShardResponse::SendMessages)
}
ShardRequestPayload::PollMessages { args, consumer } => {
let auto_commit = args.auto_commit;
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
- let (metadata, batches) = shard.streams.poll_messages(&ns, consumer, args).await?;
+
+ // Get numeric IDs for partition_store lookup
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ // Use partition_store - check for stale entries and initialize on-demand
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ // Get partition metadata from SharedMetadata (source of truth)
+ let metadata = shard.shared_metadata.load();
+ let partition_meta = metadata.partitions.get(&namespace);
+
+ // Check if partition_store entry exists AND is valid (not stale)
+ let needs_init = {
+ let store = shard.partition_store.borrow();
+ match (store.get(&namespace), partition_meta) {
+ (Some(data), Some(meta)) if data.creation_version == meta.creation_version => {
+ false
+ }
+ (Some(_), _) => {
+ // Entry exists but is stale (different version or no metadata)
+ drop(store);
+ shard.partition_store.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => true,
+ }
+ };
+
+ if needs_init {
+ // Partition not yet initialized or stale - initialize on-demand
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(IggyTimestamp::now);
+
+ shard
+ .init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ created_at,
+ )
+ .await?;
+ }
+
+ let (metadata, batches) = shard
+ .poll_messages_from_partition_store(&namespace, consumer, args)
+ .await?;
if auto_commit && !batches.is_empty() {
let offset = batches
.last_offset()
.expect("Batch set should have at least one batch");
shard
- .streams
- .auto_commit_consumer_offset(
- &shard.config.system,
- ns.stream_id(),
- ns.topic_id(),
- partition_id,
- consumer,
- offset,
- )
+ .auto_commit_consumer_offset_from_partition_store(&namespace, consumer, offset)
.await?;
}
Ok(ShardResponse::PollMessages((metadata, batches)))
@@ -108,6 +203,110 @@ async fn handle_request(
.await?;
Ok(ShardResponse::DeleteSegments)
}
+ ShardRequestPayload::CreatePartitions {
+ user_id,
+ stream_id,
+ topic_id,
+ partitions_count,
+ } => {
+ assert_eq!(
+ shard.id, 0,
+ "CreatePartitions should only be handled by shard0"
+ );
+
+ let session = Session::stateless(
+ user_id,
+ std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0),
+ );
+
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ let partition_infos = shard
+ .create_partitions(&session, &stream_id, &topic_id, partitions_count)
+ .await?;
+ let partition_ids = partition_infos.iter().map(|p| p.id).collect::<Vec<_>>();
+
+ let event = ShardEvent::CreatedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions: partition_infos,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ // Rebalance consumer groups using SharedMetadata
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+ shard.shared_metadata.rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ &partition_ids,
+ );
+
+ Ok(ShardResponse::CreatePartitionsResponse(partition_ids))
+ }
+ ShardRequestPayload::DeletePartitions {
+ user_id,
+ stream_id,
+ topic_id,
+ partitions_count,
+ } => {
+ assert_eq!(
+ shard.id, 0,
+ "DeletePartitions should only be handled by shard0"
+ );
+
+ let session = Session::stateless(
+ user_id,
+ std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0),
+ );
+
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ let deleted_partition_ids = shard
+ .delete_partitions(&session, &stream_id, &topic_id, partitions_count)
+ .await?;
+
+ let event = ShardEvent::DeletedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions_count,
+ partition_ids: deleted_partition_ids.clone(),
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ // Rebalance consumer groups using SharedMetadata
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+ // Get remaining partition IDs from SharedMetadata
+ let remaining_partition_ids: Vec<_> = shard
+ .shared_metadata
+ .load()
+ .partitions_by_topic
+ .get(&(numeric_stream_id, numeric_topic_id))
+ .map(|set| set.iter().copied().collect())
+ .unwrap_or_default();
+ shard.shared_metadata.rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ &remaining_partition_ids,
+ );
+
+ Ok(ShardResponse::DeletePartitionsResponse(
+ deleted_partition_ids,
+ ))
+ }
ShardRequestPayload::CreateStream { user_id, name } => {
assert_eq!(shard.id, 0, "CreateStream should only be handled by shard0");
@@ -119,17 +318,15 @@ async fn handle_request(
// Acquire stream lock to serialize filesystem operations
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard.create_stream(&session, name.clone()).await?;
- let created_stream_id = stream.id();
+ let created_stream_id = shard.create_stream(&session, name.clone()).await?;
let event = ShardEvent::CreatedStream {
id: created_stream_id,
- stream: stream.clone(),
};
shard.broadcast_event_to_all_shards(event).await?;
- Ok(ShardResponse::CreateStreamResponse(stream))
+ Ok(ShardResponse::CreateStreamResponse(created_stream_id))
}
ShardRequestPayload::CreateTopic {
user_id,
@@ -151,7 +348,7 @@ async fn handle_request(
// Acquire topic lock to serialize filesystem operations
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard
+ let topic_id_num = shard
.create_topic(
&session,
&stream_id,
@@ -162,30 +359,28 @@ async fn handle_request(
replication_factor,
)
.await?;
-
- let topic_id = topic.id();
let event = ShardEvent::CreatedTopic {
stream_id: stream_id.clone(),
- topic: topic.clone(),
+ topic_id: topic_id_num,
};
shard.broadcast_event_to_all_shards(event).await?;
- let partitions = shard
+ let partition_infos = shard
.create_partitions(
&session,
&stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
+ &Identifier::numeric(topic_id_num as u32).unwrap(),
partitions_count,
)
.await?;
let event = ShardEvent::CreatedPartitions {
stream_id: stream_id.clone(),
- topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions,
+ topic_id: Identifier::numeric(topic_id_num as u32).unwrap(),
+ partitions: partition_infos,
};
shard.broadcast_event_to_all_shards(event).await?;
- Ok(ShardResponse::CreateTopicResponse(topic))
+ Ok(ShardResponse::CreateTopicResponse(topic_id_num))
}
ShardRequestPayload::UpdateTopic {
user_id,
@@ -241,8 +436,8 @@ async fn handle_request(
);
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard.delete_topic(&session, &stream_id, &topic_id).await?;
- let topic_id_num = topic.root().id();
+ let topic_info = shard.delete_topic(&session, &stream_id, &topic_id).await?;
+ let topic_id_num = topic_info.id;
let event = ShardEvent::DeletedTopic {
id: topic_id_num,
@@ -251,7 +446,7 @@ async fn handle_request(
};
shard.broadcast_event_to_all_shards(event).await?;
- Ok(ShardResponse::DeleteTopicResponse(topic))
+ Ok(ShardResponse::DeleteTopicResponse(topic_id_num))
}
ShardRequestPayload::CreateUser {
user_id,
@@ -270,16 +465,6 @@ async fn handle_request(
let user =
shard.create_user(&session, &username, &password, status, permissions.clone())?;
- let created_user_id = user.id;
-
- let event = ShardEvent::CreatedUser {
- user_id: created_user_id,
- username: username.clone(),
- password: password.clone(),
- status,
- permissions: permissions.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
Ok(ShardResponse::CreateUserResponse(user))
}
ShardRequestPayload::GetStats { .. } => {
@@ -299,10 +484,30 @@ async fn handle_request(
);
let _user_guard = shard.fs_locks.user_lock.lock().await;
let user = shard.delete_user(&session, &user_id)?;
- let event = ShardEvent::DeletedUser { user_id };
- shard.broadcast_event_to_all_shards(event).await?;
Ok(ShardResponse::DeletedUser(user))
}
+ ShardRequestPayload::UpdateStream {
+ user_id,
+ stream_id,
+ name,
+ } => {
+ assert_eq!(shard.id, 0, "UpdateStream should only be handled by shard0");
+
+ let session = Session::stateless(
+ user_id,
+ std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0),
+ );
+
+ shard.update_stream(&session, &stream_id, name.clone())?;
+
+ let event = ShardEvent::UpdatedStream {
+ stream_id: stream_id.clone(),
+ name,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ Ok(ShardResponse::UpdateStreamResponse)
+ }
ShardRequestPayload::DeleteStream { user_id, stream_id } => {
assert_eq!(shard.id, 0, "DeleteStream should only be handled by shard0");
@@ -311,13 +516,14 @@ async fn handle_request(
std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0),
);
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard.delete_stream(&session, &stream_id).await?;
+ let stream_info = shard.delete_stream(&session, &stream_id).await?;
+ let stream_id_num = stream_info.id;
let event = ShardEvent::DeletedStream {
- id: stream.id(),
+ id: stream_id_num,
stream_id,
};
shard.broadcast_event_to_all_shards(event).await?;
- Ok(ShardResponse::DeleteStreamResponse(stream))
+ Ok(ShardResponse::DeleteStreamResponse(stream_id_num))
}
ShardRequestPayload::SocketTransfer {
fd,
@@ -351,13 +557,44 @@ async fn handle_request(
let registry = shard.task_registry.clone();
let registry_clone = registry.clone();
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
let batch = shard.maybe_encrypt_messages(initial_data)?;
let messages_count = batch.count();
+ // Get numeric IDs for partition_store lookup
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
+
+ // Use partition_store - initialize on-demand if not yet ready
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ if !shard.partition_store.borrow().contains(&namespace) {
+ // Partition not yet initialized (race with CreatedPartitions event)
+ // Initialize it on-demand using SharedMetadata
+ let created_at = shard
+ .shared_metadata
+ .load()
+ .partitions
+ .get(&namespace)
+ .map(|meta| meta.created_at)
+ .unwrap_or_else(IggyTimestamp::now);
+
+ shard
+ .init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ created_at,
+ )
+ .await?;
+ }
+
shard
- .streams
- .append_messages(&shard.config.system, &shard.task_registry, &ns, batch)
+ .append_messages_to_partition_store(&namespace, batch, &shard.config.system)
.await?;
shard.metrics.increment_messages(messages_count as u64);
@@ -408,16 +645,47 @@ pub async fn handle_event(shard: &Rc<IggyShard>, event: ShardEvent) -> Result<()
partitions_count,
partition_ids,
} => {
- shard.delete_partitions_bypass_auth(
- &stream_id,
- &topic_id,
- partitions_count,
- partition_ids,
- )?;
+ if shard.id == 0 {
+ // On shard-0, update the slab
+ shard.delete_partitions_bypass_auth(
+ &stream_id,
+ &topic_id,
+ partitions_count,
+ partition_ids.clone(),
+ )?;
+ }
+ // On all shards, clean up local partition_store entries
+ let numeric_stream_id = shard
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .unwrap_or_default();
+ let numeric_topic_id = shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .unwrap_or_default();
+ let mut store = shard.partition_store.borrow_mut();
+ for partition_id in partition_ids {
+ let ns = crate::shard::namespace::IggyNamespace::new(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ );
+ store.remove(&ns);
+ }
Ok(())
}
ShardEvent::UpdatedStream { stream_id, name } => {
- shard.update_stream_bypass_auth(&stream_id, &name)?;
+ if shard.id == 0 {
+ // Shard-0 updates via the request handler, but we update SharedMetadata here for non-shard-0
+ shard.update_stream_bypass_auth(&stream_id, &name)?;
+ } else {
+ // Non-shard-0: only update SharedMetadata (if stream still exists)
+ if let Some(numeric_stream_id) = shard.shared_metadata.get_stream_id(&stream_id) {
+ shard
+ .shared_metadata
+ .update_stream(numeric_stream_id, Arc::from(name.as_str()));
+ }
+ }
Ok(())
}
ShardEvent::PurgedStream { stream_id } => {
@@ -431,59 +699,6 @@ pub async fn handle_event(shard: &Rc<IggyShard>, event: ShardEvent) -> Result<()
shard.purge_topic_bypass_auth(&stream_id, &topic_id).await?;
Ok(())
}
- ShardEvent::CreatedUser {
- user_id,
- username,
- password,
- status,
- permissions,
- } => {
- shard.create_user_bypass_auth(
- user_id,
- &username,
- &password,
- status,
- permissions.clone(),
- )?;
- Ok(())
- }
- ShardEvent::DeletedUser { user_id } => {
- shard.delete_user_bypass_auth(&user_id)?;
- Ok(())
- }
- ShardEvent::ChangedPassword {
- user_id,
- current_password,
- new_password,
- } => {
- shard.change_password_bypass_auth(&user_id, ¤t_password, &new_password)?;
- Ok(())
- }
- ShardEvent::CreatedPersonalAccessToken {
- personal_access_token,
- } => {
- shard.create_personal_access_token_bypass_auth(personal_access_token.to_owned())?;
- Ok(())
- }
- ShardEvent::DeletedPersonalAccessToken { user_id, name } => {
- shard.delete_personal_access_token_bypass_auth(user_id, &name)?;
- Ok(())
- }
- ShardEvent::UpdatedUser {
- user_id,
- username,
- status,
- } => {
- shard.update_user_bypass_auth(&user_id, username.to_owned(), status)?;
- Ok(())
- }
- ShardEvent::UpdatedPermissions {
- user_id,
- permissions,
- } => {
- shard.update_permissions_bypass_auth(&user_id, permissions.to_owned())?;
- Ok(())
- }
ShardEvent::AddressBound { protocol, address } => {
info!(
"Received AddressBound event for {:?} with address: {}",
@@ -509,92 +724,157 @@ pub async fn handle_event(shard: &Rc<IggyShard>, event: ShardEvent) -> Result<()
}
Ok(())
}
- ShardEvent::CreatedStream { id, stream } => {
- let stream_id = shard.create_stream_bypass_auth(stream);
- assert_eq!(stream_id, id);
- Ok(())
- }
- ShardEvent::DeletedStream { id, stream_id } => {
- let stream = shard.delete_stream_bypass_auth(&stream_id);
- assert_eq!(stream.id(), id);
-
+ // ========================================================================
+ // METADATA EVENTS - SharedMetadata is already updated by originating shard
+ // These events are no-ops: we don't maintain slab state from events anymore
+ // ========================================================================
+ ShardEvent::CreatedStream { id: _ } => {
+ // SharedMetadata already has this stream - no action needed
Ok(())
}
- ShardEvent::CreatedTopic { stream_id, topic } => {
- let topic_id_from_event = topic.id();
- let topic_id = shard.create_topic_bypass_auth(&stream_id, topic.clone());
- assert_eq!(topic_id, topic_id_from_event);
+ ShardEvent::DeletedStream {
+ id: _,
+ stream_id: _,
+ } => {
+ // SharedMetadata already deleted this stream - no action needed
Ok(())
}
- ShardEvent::CreatedPartitions {
- stream_id,
- topic_id,
- partitions,
+ ShardEvent::CreatedTopic {
+ stream_id: _,
+ topic_id: _,
} => {
- shard
- .create_partitions_bypass_auth(&stream_id, &topic_id, partitions)
- .await?;
+ // SharedMetadata already has this topic - no action needed
Ok(())
}
ShardEvent::DeletedTopic {
- id,
- stream_id,
- topic_id,
+ id: _,
+ stream_id: _,
+ topic_id: _,
} => {
- let topic = shard.delete_topic_bypass_auth(&stream_id, &topic_id);
- assert_eq!(topic.id(), id);
+ // SharedMetadata already deleted this topic - no action needed
Ok(())
}
ShardEvent::UpdatedTopic {
- stream_id,
- topic_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
+ stream_id: _,
+ topic_id: _,
+ name: _,
+ message_expiry: _,
+ compression_algorithm: _,
+ max_topic_size: _,
+ replication_factor: _,
} => {
- shard.update_topic_bypass_auth(
- &stream_id,
- &topic_id,
- name.clone(),
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )?;
+ // SharedMetadata already updated this topic - no action needed
Ok(())
}
ShardEvent::CreatedConsumerGroup {
- stream_id,
- topic_id,
- cg,
+ stream_id: _,
+ topic_id: _,
+ cg_id: _,
} => {
- let cg_id = cg.id();
- let id = shard.create_consumer_group_bypass_auth(&stream_id, &topic_id, cg);
- assert_eq!(id, cg_id);
+ // SharedMetadata already has this consumer group - no action needed
Ok(())
}
ShardEvent::DeletedConsumerGroup {
- id,
+ id: _,
+ stream_id: _,
+ topic_id: _,
+ group_id: _,
+ } => {
+ // SharedMetadata already deleted this consumer group - no action needed
+ Ok(())
+ }
+
+ // ========================================================================
+ // PARTITION EVENTS - Need to initialize logs in partition_store
+ // ========================================================================
+ ShardEvent::CreatedPartitions {
stream_id,
topic_id,
- group_id,
+ partitions,
} => {
- let cg = shard.delete_consumer_group_bypass_auth(&stream_id, &topic_id, &group_id);
- assert_eq!(cg.id(), id);
+ // Get numeric IDs from SharedMetadata (already updated by originating shard)
+ let numeric_stream_id = match shard.shared_metadata.get_stream_id(&stream_id) {
+ Some(id) => id,
+ None => {
+ tracing::warn!(
+ "CreatedPartitions: stream {:?} not found in SharedMetadata",
+ stream_id
+ );
+ return Ok(());
+ }
+ };
+ let numeric_topic_id = match shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ {
+ Some(id) => id,
+ None => {
+ tracing::warn!(
+ "CreatedPartitions: topic {:?} not found in SharedMetadata for stream {}",
+ topic_id,
+ numeric_stream_id
+ );
+ return Ok(());
+ }
+ };
+ // Initialize logs directly in partition_store for partitions owned by this shard
+ let shards_count = shard.get_available_shards_count();
+ for partition_info in partitions {
+ let ns = crate::shard::namespace::IggyNamespace::new(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_info.id,
+ );
+ let owner_shard_id = crate::shard::calculate_shard_assignment(&ns, shards_count);
+
+ if shard.id == owner_shard_id as u16 {
+ shard
+ .init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_info.id,
+ partition_info.created_at,
+ )
+ .await?;
+ }
+ }
Ok(())
}
+
+ // ========================================================================
+ // OPERATIONAL EVENTS - Work directly with partition_store
+ // ========================================================================
ShardEvent::FlushUnsavedBuffer {
stream_id,
topic_id,
partition_id,
fsync,
} => {
- shard
- .flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync)
- .await?;
+ // Get numeric IDs from SharedMetadata
+ let numeric_stream_id = match shard.shared_metadata.get_stream_id(&stream_id) {
+ Some(id) => id,
+ None => return Ok(()), // Stream gone, nothing to flush
+ };
+ let numeric_topic_id = match shard
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ {
+ Some(id) => id,
+ None => return Ok(()), // Topic gone, nothing to flush
+ };
+
+ // Flush directly from partition_store
+ let ns = crate::shard::namespace::IggyNamespace::new(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ );
+ if shard.partition_store.borrow().get(&ns).is_some() {
+ shard
+ .flush_unsaved_buffer_from_partition_store(&ns, fsync)
+ .await?;
+ }
Ok(())
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 8939e87b0..736413a42 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -18,6 +18,7 @@ inner() * or more contributor license agreements. See the NOTICE file
pub mod builder;
pub mod namespace;
+pub mod partition_store;
pub mod system;
pub mod task_registry;
pub mod tasks;
@@ -33,8 +34,11 @@ use self::tasks::{continuous, periodic};
use crate::{
configs::server::ServerConfig,
io::fs_locks::FsLocks,
- shard::{task_registry::TaskRegistry, transmission::frame::ShardFrame},
- slab::{streams::Streams, traits_ext::EntityMarker, users::Users},
+ metadata::{SharedConsumerOffsetsStore, SharedMetadata},
+ shard::{
+ namespace::IggyNamespace, partition_store::PartitionDataStore, task_registry::TaskRegistry,
+ transmission::frame::ShardFrame,
+ },
state::file::FileState,
streaming::{
clients::client_manager::ClientManager, diagnostics::metrics::Metrics, session::Session,
@@ -44,13 +48,16 @@ use crate::{
};
use builder::IggyShardBuilder;
use dashmap::DashMap;
-use iggy_common::sharding::{IggyNamespace, PartitionLocation};
-use iggy_common::{EncryptorKind, Identifier, IggyError};
+use iggy_common::sharding::PartitionLocation;
+use iggy_common::{EncryptorKind, IggyError};
use std::{
cell::{Cell, RefCell},
net::SocketAddr,
rc::Rc,
- sync::atomic::{AtomicBool, Ordering},
+ sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicU64, Ordering},
+ },
time::{Duration, Instant},
};
use tracing::{debug, error, info, instrument};
@@ -65,7 +72,15 @@ pub struct IggyShard {
shards: Vec<ShardConnector<ShardFrame>>,
_version: SemanticVersion,
- pub(crate) streams: Streams,
+ // Shared metadata structures (source of truth)
+ pub(crate) shared_metadata: &'static SharedMetadata,
+ pub(crate) shared_consumer_offsets: &'static SharedConsumerOffsetsStore,
+ pub(crate) partition_store: RefCell<PartitionDataStore>,
+ /// Tracks partitions currently being initialized to prevent TOCTOU races.
+ /// In single-threaded async, multiple tasks could see needs_init=true before
+ /// any completes initialization. This set ensures only one task initializes.
+ pub(crate) pending_partition_inits: RefCell<std::collections::HashSet<IggyNamespace>>,
+
pub(crate) shards_table: EternalPtr<DashMap<IggyNamespace, PartitionLocation>>,
pub(crate) state: FileState,
@@ -73,8 +88,7 @@ pub struct IggyShard {
pub(crate) encryptor: Option<EncryptorKind>,
pub(crate) config: ServerConfig,
pub(crate) client_manager: ClientManager,
- pub(crate) permissioner: RefCell<Permissioner>,
- pub(crate) users: Users,
+ pub(crate) permissioner: Permissioner,
pub(crate) metrics: Metrics,
pub(crate) is_follower: bool,
pub messages_receiver: Cell<Option<Receiver<ShardFrame>>>,
@@ -185,6 +199,8 @@ impl IggyShard {
async fn load_segments(&self) -> Result<(), IggyError> {
use crate::bootstrap::load_segments;
+ use crate::shard::partition_store::PartitionData;
+
for shard_entry in self.shards_table.iter() {
let (namespace, location) = shard_entry.pair();
@@ -202,33 +218,97 @@ impl IggyShard {
self.config
.system
.get_partition_path(stream_id, topic_id, partition_id);
- let stats = self.streams.with_partition_by_id(
- &Identifier::numeric(stream_id as u32).unwrap(),
- &Identifier::numeric(topic_id as u32).unwrap(),
- partition_id,
- |(_, stats, ..)| stats.clone(),
+
+ // Get partition metadata and stats from SharedMetadata
+ let metadata = self.shared_metadata.load();
+ let partition_meta = metadata
+ .partitions
+ .get(namespace)
+ .expect("Partition must exist in SharedMetadata");
+ let created_at = partition_meta.created_at;
+ let stats = partition_meta.stats.clone();
+ drop(metadata);
+
+ // Load consumer offsets from disk
+ use crate::streaming::partitions::helpers::create_message_deduplicator;
+ use crate::streaming::partitions::storage::{
+ load_consumer_group_offsets, load_consumer_offsets,
+ };
+ use ahash::HashMap;
+
+ let consumer_offset_path =
+ self.config
+ .system
+ .get_consumer_offsets_path(stream_id, topic_id, partition_id);
+ let consumer_group_offsets_path = self
+ .config
+ .system
+ .get_consumer_group_offsets_path(stream_id, topic_id, partition_id);
+
+ let consumer_offsets = Arc::new(
+ load_consumer_offsets(&consumer_offset_path)
+ .unwrap_or_default()
+ .into_iter()
+ .map(|offset| (offset.consumer_id as usize, offset))
+ .collect::<HashMap<usize, _>>()
+ .into(),
);
+
+ let consumer_group_offsets = Arc::new(
+ load_consumer_group_offsets(&consumer_group_offsets_path)
+ .unwrap_or_default()
+ .into_iter()
+ .collect::<HashMap<_, _>>()
+ .into(),
+ );
+
+ let message_deduplicator =
+ create_message_deduplicator(&self.config.system).map(Arc::new);
+
+ // Check if partition has data (for should_increment_offset)
+ let log_files_exist = std::path::Path::new(&partition_path).exists();
+ let should_increment_offset = log_files_exist;
+
match load_segments(
&self.config.system,
stream_id,
topic_id,
partition_id,
partition_path,
- stats,
+ stats.clone(),
)
.await
{
Ok(loaded_log) => {
- self.streams.with_partition_by_id_mut(
- &Identifier::numeric(stream_id as u32).unwrap(),
- &Identifier::numeric(topic_id as u32).unwrap(),
- partition_id,
- |(_, _, _, offset, .., log)| {
- *log = loaded_log;
- let current_offset = log.active_segment().end_offset;
- offset.store(current_offset, Ordering::Relaxed);
- },
+ let current_offset = loaded_log.active_segment().end_offset;
+ stats.set_current_offset(current_offset);
+
+ // Get creation_version from SharedMetadata (source of truth)
+ let creation_version = self
+ .shared_metadata
+ .load()
+ .partitions
+ .get(namespace)
+ .map(|meta| meta.creation_version)
+ .unwrap_or(0);
+
+ // Create PartitionData and insert into partition_store
+ let partition_data = PartitionData::with_log(
+ loaded_log,
+ stats,
+ Arc::new(AtomicU64::new(current_offset)),
+ consumer_offsets,
+ consumer_group_offsets,
+ message_deduplicator,
+ created_at,
+ creation_version,
+ should_increment_offset,
);
+
+ self.partition_store
+ .borrow_mut()
+ .insert(*namespace, partition_data);
+
info!(
"Successfully loaded segments for stream: {}, topic: {}, partition: {}",
stream_id, topic_id, partition_id
@@ -249,11 +329,8 @@ impl IggyShard {
}
async fn load_users(&self) -> Result<(), IggyError> {
- let users_list = self.users.values();
- let users_count = users_list.len();
- self.permissioner
- .borrow_mut()
- .init(&users_list.iter().collect::<Vec<_>>());
+ // Permissioner reads directly from SharedMetadata, no initialization needed
+ let users_count = self.shared_metadata.users_count();
self.metrics.increment_users(users_count as u32);
info!("Initialized {} user(s).", users_count);
Ok(())
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index 289587179..392fa2b15 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -16,18 +16,20 @@
* under the License.
*/
-use crate::slab::partitions;
use iggy_common::Identifier;
+// Re-export IggyNamespace from iggy_common for backwards compatibility
+pub use iggy_common::sharding::IggyNamespace;
+
#[derive(Debug)]
pub struct IggyFullNamespace {
stream: Identifier,
topic: Identifier,
- partition: partitions::ContainerId,
+ partition: usize,
}
impl IggyFullNamespace {
- pub fn new(stream: Identifier, topic: Identifier, partition: partitions::ContainerId) -> Self {
+ pub fn new(stream: Identifier, topic: Identifier, partition: usize) -> Self {
Self {
stream,
topic,
@@ -43,11 +45,11 @@ impl IggyFullNamespace {
&self.topic
}
- pub fn partition_id(&self) -> partitions::ContainerId {
+ pub fn partition_id(&self) -> usize {
self.partition
}
- pub fn decompose(self) -> (Identifier, Identifier, partitions::ContainerId) {
+ pub fn decompose(self) -> (Identifier, Identifier, usize) {
(self.stream, self.topic, self.partition)
}
}
diff --git a/core/server/src/shard/partition_store.rs b/core/server/src/shard/partition_store.rs
new file mode 100644
index 000000000..b5331fb83
--- /dev/null
+++ b/core/server/src/shard/partition_store.rs
@@ -0,0 +1,288 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Per-shard partition data storage.
+//!
+//! Each shard runs on a single-threaded compio runtime, so per-shard data
+//! needs NO synchronization. This uses a plain HashMap for maximum performance.
+
+use super::namespace::IggyNamespace;
+use crate::streaming::{
+ deduplication::message_deduplicator::MessageDeduplicator,
+ partitions::{
+ journal::MemoryMessageJournal,
+ log::SegmentedLog,
+ partition::{ConsumerGroupOffsets, ConsumerOffsets},
+ },
+ stats::PartitionStats,
+};
+use iggy_common::IggyTimestamp;
+use std::{
+ collections::HashMap,
+ sync::{Arc, atomic::AtomicU64},
+};
+
+/// Per-shard partition data - mutable, single-threaded access.
+#[derive(Debug)]
+pub struct PartitionData {
+ pub log: SegmentedLog<MemoryMessageJournal>,
+ pub offset: Arc<AtomicU64>,
+ pub consumer_offsets: Arc<ConsumerOffsets>,
+ pub consumer_group_offsets: Arc<ConsumerGroupOffsets>,
+ pub message_deduplicator: Option<Arc<MessageDeduplicator>>,
+ pub stats: Arc<PartitionStats>,
+ pub created_at: IggyTimestamp,
+ /// Monotonically increasing version from SharedMetadata to detect stale entries.
+ pub creation_version: u64,
+ pub should_increment_offset: bool,
+}
+
+impl PartitionData {
+ /// Create new partition data with default log.
+ #[allow(clippy::too_many_arguments)]
+ pub fn new(
+ stats: Arc<PartitionStats>,
+ offset: Arc<AtomicU64>,
+ consumer_offsets: Arc<ConsumerOffsets>,
+ consumer_group_offsets: Arc<ConsumerGroupOffsets>,
+ message_deduplicator: Option<Arc<MessageDeduplicator>>,
+ created_at: IggyTimestamp,
+ creation_version: u64,
+ should_increment_offset: bool,
+ ) -> Self {
+ Self {
+ log: SegmentedLog::default(),
+ offset,
+ consumer_offsets,
+ consumer_group_offsets,
+ message_deduplicator,
+ stats,
+ created_at,
+ creation_version,
+ should_increment_offset,
+ }
+ }
+
+ /// Create partition data with existing log (e.g., loaded from disk).
+ #[allow(clippy::too_many_arguments)]
+ pub fn with_log(
+ log: SegmentedLog<MemoryMessageJournal>,
+ stats: Arc<PartitionStats>,
+ offset: Arc<AtomicU64>,
+ consumer_offsets: Arc<ConsumerOffsets>,
+ consumer_group_offsets: Arc<ConsumerGroupOffsets>,
+ message_deduplicator: Option<Arc<MessageDeduplicator>>,
+ created_at: IggyTimestamp,
+ creation_version: u64,
+ should_increment_offset: bool,
+ ) -> Self {
+ Self {
+ log,
+ offset,
+ consumer_offsets,
+ consumer_group_offsets,
+ message_deduplicator,
+ stats,
+ created_at,
+ creation_version,
+ should_increment_offset,
+ }
+ }
+}
+
+/// Per-shard partition data storage.
+/// Single-threaded (compio runtime) - NO synchronization needed!
+#[derive(Debug, Default)]
+pub struct PartitionDataStore {
+ partitions: HashMap<IggyNamespace, PartitionData>,
+}
+
+impl PartitionDataStore {
+ pub fn new() -> Self {
+ Self {
+ partitions: HashMap::new(),
+ }
+ }
+
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ partitions: HashMap::with_capacity(capacity),
+ }
+ }
+
+ #[inline]
+ pub fn get(&self, ns: &IggyNamespace) -> Option<&PartitionData> {
+ self.partitions.get(ns)
+ }
+
+ #[inline]
+ pub fn get_mut(&mut self, ns: &IggyNamespace) -> Option<&mut PartitionData> {
+ self.partitions.get_mut(ns)
+ }
+
+ #[inline]
+ pub fn insert(&mut self, ns: IggyNamespace, data: PartitionData) {
+ self.partitions.insert(ns, data);
+ }
+
+ #[inline]
+ pub fn remove(&mut self, ns: &IggyNamespace) -> Option<PartitionData> {
+ self.partitions.remove(ns)
+ }
+
+ #[inline]
+ pub fn contains(&self, ns: &IggyNamespace) -> bool {
+ self.partitions.contains_key(ns)
+ }
+
+ #[inline]
+ pub fn len(&self) -> usize {
+ self.partitions.len()
+ }
+
+ #[inline]
+ pub fn is_empty(&self) -> bool {
+ self.partitions.is_empty()
+ }
+
+ /// Iterate over all namespaces owned by this shard.
+ pub fn namespaces(&self) -> impl Iterator<Item = &IggyNamespace> {
+ self.partitions.keys()
+ }
+
+ /// Iterate over all partition data.
+ pub fn iter(&self) -> impl Iterator<Item = (&IggyNamespace, &PartitionData)> {
+ self.partitions.iter()
+ }
+
+ /// Iterate over all partition data mutably.
+ pub fn iter_mut(&mut self) -> impl Iterator<Item = (&IggyNamespace, &mut PartitionData)> {
+ self.partitions.iter_mut()
+ }
+
+ /// Remove multiple partitions at once.
+ pub fn remove_many(&mut self, namespaces: &[IggyNamespace]) -> Vec<PartitionData> {
+ namespaces
+ .iter()
+ .filter_map(|ns| self.partitions.remove(ns))
+ .collect()
+ }
+
+ /// Get partition data, initializing if not present.
+ /// Returns None if initialization fails.
+ pub fn get_or_init<F>(&mut self, ns: IggyNamespace, init: F) -> &mut PartitionData
+ where
+ F: FnOnce() -> PartitionData,
+ {
+ self.partitions.entry(ns).or_insert_with(init)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::streaming::stats::{StreamStats, TopicStats};
+
+ fn create_test_partition_data() -> PartitionData {
+ let stream_stats = Arc::new(StreamStats::default());
+ let topic_stats = Arc::new(TopicStats::new(stream_stats));
+ let partition_stats = Arc::new(PartitionStats::new(topic_stats));
+
+ PartitionData::new(
+ partition_stats,
+ Arc::new(AtomicU64::new(0)),
+ Arc::new(ConsumerOffsets::with_capacity(10)),
+ Arc::new(ConsumerGroupOffsets::with_capacity(10)),
+ None,
+ IggyTimestamp::now(),
+ 1, // creation_version
+ true,
+ )
+ }
+
+ #[test]
+ fn test_basic_operations() {
+ let mut store = PartitionDataStore::new();
+ let ns = IggyNamespace::new(1, 1, 0);
+
+ assert!(!store.contains(&ns));
+ assert!(store.is_empty());
+
+ store.insert(ns, create_test_partition_data());
+
+ assert!(store.contains(&ns));
+ assert_eq!(store.len(), 1);
+ assert!(store.get(&ns).is_some());
+ assert!(store.get_mut(&ns).is_some());
+
+ let removed = store.remove(&ns);
+ assert!(removed.is_some());
+ assert!(!store.contains(&ns));
+ assert!(store.is_empty());
+ }
+
+ #[test]
+ fn test_iteration() {
+ let mut store = PartitionDataStore::new();
+ let ns1 = IggyNamespace::new(1, 1, 0);
+ let ns2 = IggyNamespace::new(1, 1, 1);
+ let ns3 = IggyNamespace::new(1, 2, 0);
+
+ store.insert(ns1, create_test_partition_data());
+ store.insert(ns2, create_test_partition_data());
+ store.insert(ns3, create_test_partition_data());
+
+ let namespaces: Vec<_> = store.namespaces().collect();
+ assert_eq!(namespaces.len(), 3);
+
+ let pairs: Vec<_> = store.iter().collect();
+ assert_eq!(pairs.len(), 3);
+ }
+
+ #[test]
+ fn test_remove_many() {
+ let mut store = PartitionDataStore::new();
+ let ns1 = IggyNamespace::new(1, 1, 0);
+ let ns2 = IggyNamespace::new(1, 1, 1);
+ let ns3 = IggyNamespace::new(1, 2, 0);
+
+ store.insert(ns1, create_test_partition_data());
+ store.insert(ns2, create_test_partition_data());
+ store.insert(ns3, create_test_partition_data());
+
+ let removed = store.remove_many(&[ns1, ns2]);
+ assert_eq!(removed.len(), 2);
+ assert!(!store.contains(&ns1));
+ assert!(!store.contains(&ns2));
+ assert!(store.contains(&ns3));
+ }
+
+ #[test]
+ fn test_get_or_init() {
+ let mut store = PartitionDataStore::new();
+ let ns = IggyNamespace::new(1, 1, 0);
+
+ assert!(!store.contains(&ns));
+
+ let _ = store.get_or_init(ns, create_test_partition_data);
+ assert!(store.contains(&ns));
+
+ // Second call should not reinitialize
+ let data = store.get_or_init(ns, || panic!("Should not be called"));
+ assert!(data.should_increment_offset);
+ }
+}
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index 2259e5b11..ab57dd6f0 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -75,7 +75,6 @@ impl IggyShard {
) -> Result<Option<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
- .borrow()
.get_client(session.get_user_id())
.error(|e: &IggyError| {
format!(
@@ -90,7 +89,6 @@ impl IggyShard {
pub fn get_clients(&self, session: &Session) -> Result<Vec<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
- .borrow()
.get_clients(session.get_user_id())
.error(|e: &IggyError| {
format!(
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index 1f9610a22..dd55ae88d 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -17,95 +17,92 @@
*/
use super::COMPONENT;
+use crate::metadata::ConsumerGroupMeta;
use crate::shard::IggyShard;
-use crate::slab::consumer_groups;
-use crate::slab::traits_ext::EntityMarker;
-use crate::slab::traits_ext::Insert;
-use crate::streaming::partitions;
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use crate::streaming::topics;
-use crate::streaming::topics::consumer_group;
-use crate::streaming::topics::consumer_group::MEMBERS_CAPACITY;
-use arcshift::ArcShift;
use err_trail::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use slab::Slab;
+use imbl::Vector as ImVector;
+use std::sync::Arc;
impl IggyShard {
+ /// Creates a consumer group and returns the consumer group ID.
+ /// Writes only to SharedMetadata (source of truth).
pub fn create_consumer_group(
&self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
name: String,
- ) -> Result<consumer_group::ConsumerGroup, IggyError> {
+ ) -> Result<usize, IggyError> {
self.ensure_authenticated(session)?;
self.ensure_topic_exists(stream_id, topic_id)?;
- let exists = self
- .streams
- .with_topic_by_id(stream_id, topic_id, |(root, ..)| {
- root.consumer_groups()
- .exists(&name.clone().try_into().unwrap())
- });
- if exists {
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream existence already verified");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic existence already verified");
+
+ // Check consumer group existence using SharedMetadata
+ if self.shared_metadata.consumer_group_exists_by_name(
+ numeric_stream_id,
+ numeric_topic_id,
+ &name,
+ ) {
return Err(IggyError::ConsumerGroupNameAlreadyExists(
name,
topic_id.clone(),
));
}
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().create_consumer_group(
- session.get_user_id(),
- stream_id,
- topic_id,
- ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?;
- }
- let cg = self.create_and_insert_consumer_group_mem(stream_id, topic_id, name);
- Ok(cg)
- }
+ self.permissioner
+ .create_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id
+ )
+ })?;
- fn create_and_insert_consumer_group_mem(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- name: String,
- ) -> consumer_group::ConsumerGroup {
- let partitions = self.streams.with_topics(stream_id, |topics| {
- topics.with_partitions(topic_id, partitions::helpers::get_partition_ids())
- });
- let members = ArcShift::new(Slab::with_capacity(MEMBERS_CAPACITY));
- let mut cg = consumer_group::ConsumerGroup::new(name, members, partitions);
- let id = self.insert_consumer_group_mem(stream_id, topic_id, cg.clone());
- cg.update_id(id);
- cg
- }
+ // Get partition IDs from SharedMetadata (works on all shards)
+ // Must sort to ensure deterministic partition order for round-robin
+ let mut partitions: Vec<usize> = self
+ .shared_metadata
+ .load()
+ .partitions_by_topic
+ .get(&(numeric_stream_id, numeric_topic_id))
+ .map(|set| set.iter().copied().collect())
+ .unwrap_or_default();
+ partitions.sort_unstable();
- fn insert_consumer_group_mem(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- cg: consumer_group::ConsumerGroup,
- ) -> consumer_groups::ContainerId {
- self.streams
- .with_consumer_groups_mut(stream_id, topic_id, |container| container.insert(cg))
- }
+ // Generate next ID from SharedMetadata
+ let id = self
+ .shared_metadata
+ .next_consumer_group_id(numeric_stream_id, numeric_topic_id);
- pub fn create_consumer_group_bypass_auth(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- cg: consumer_group::ConsumerGroup,
- ) -> usize {
- self.insert_consumer_group_mem(stream_id, topic_id, cg)
+ // Write to SharedMetadata only (source of truth)
+ let key = (numeric_stream_id, numeric_topic_id, id);
+ self.shared_metadata.add_consumer_group(
+ key,
+ ConsumerGroupMeta {
+ id,
+ stream_id: numeric_stream_id,
+ topic_id: numeric_topic_id,
+ name: Arc::from(name.as_str()),
+ partitions,
+ members: ImVector::new(),
+ },
+ );
+
+ Ok(id)
}
pub fn delete_consumer_group(
@@ -114,22 +111,31 @@ impl IggyShard {
stream_id: &Identifier,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> Result<consumer_group::ConsumerGroup, IggyError> {
+ ) -> Result<ConsumerGroupMeta, IggyError> {
self.ensure_authenticated(session)?;
self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().delete_consumer_group(
- session.get_user_id(),
- stream_id,
- topic_id,
- ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to delete consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?;
- }
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream existence already verified");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic existence already verified");
+
+ self.permissioner
+ .delete_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - permission denied to delete consumer group for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id
+ )
+ })?;
+
let cg = self.delete_consumer_group_base(stream_id, topic_id, group_id);
Ok(cg)
}
@@ -139,7 +145,7 @@ impl IggyShard {
stream_id: &Identifier,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> consumer_group::ConsumerGroup {
+ ) -> ConsumerGroupMeta {
self.delete_consumer_group_base(stream_id, topic_id, group_id)
}
@@ -148,32 +154,61 @@ impl IggyShard {
stream_id: &Identifier,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> consumer_group::ConsumerGroup {
- // Get numeric IDs before deletion for ClientManager cleanup
- let stream_id_value = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- let topic_id_value =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let group_id_value = self.streams.with_consumer_group_by_id(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::get_consumer_group_id(),
+ ) -> ConsumerGroupMeta {
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+ let numeric_group_id = self
+ .shared_metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, group_id)
+ .expect("Consumer group must exist");
+
+ // Get data from SharedMetadata before deletion for return value
+ let key = (numeric_stream_id, numeric_topic_id, numeric_group_id);
+ let cg_meta = {
+ let metadata = self.shared_metadata.load();
+ metadata
+ .consumer_groups
+ .get(&key)
+ .cloned()
+ .unwrap_or_else(|| ConsumerGroupMeta {
+ id: numeric_group_id,
+ stream_id: numeric_stream_id,
+ topic_id: numeric_topic_id,
+ name: Arc::from(""),
+ partitions: Vec::new(),
+ members: ImVector::new(),
+ })
+ };
+
+ // Clean up ClientManager state
+ self.client_manager.delete_consumer_group(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
);
- let cg = self.streams.with_consumer_groups_mut(
- stream_id,
- topic_id,
- topics::helpers::delete_consumer_group(group_id),
+ // Clean up member partition indices before deleting the group
+ self.shared_metadata.remove_all_member_partition_indices(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
);
- // Clean up ClientManager state
- self.client_manager
- .delete_consumer_group(stream_id_value, topic_id_value, group_id_value);
+ // Update SharedMetadata (source of truth)
+ self.shared_metadata.delete_consumer_group(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
+ );
- cg
+ cg_meta
}
pub fn join_consumer_group(
@@ -185,53 +220,57 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().join_consumer_group(
- session.get_user_id(),
- stream_id,
- topic_id,
- ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to join consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?;
- }
+
+ // Use SharedMetadata for ID resolution (works on all shards)
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream existence already verified");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic existence already verified");
+ let numeric_group_id = self
+ .shared_metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, group_id)
+ .expect("Consumer group existence already verified");
+
+ self.permissioner
+ .join_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - permission denied to join consumer group for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id
+ )
+ })?;
+
let client_id = session.client_id;
- self.streams.with_consumer_group_by_id_mut(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::join_consumer_group(client_id),
- );
- // Update ClientManager state
- let stream_id_value = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- let topic_id_value =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let group_id_value = self.streams.with_consumer_group_by_id(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::get_consumer_group_id(),
+ // Update SharedMetadata (source of truth) - thread-safe via rcu()
+ self.shared_metadata.join_consumer_group(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
+ client_id,
);
- self.client_manager.join_consumer_group(
- session.client_id,
- stream_id_value,
- topic_id_value,
- group_id_value,
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}",
- session.client_id
+ // Update ClientManager
+ self.client_manager
+ .join_consumer_group(
+ client_id,
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
)
- })?;
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}",
+ client_id
+ )
+ })?;
+
Ok(())
}
@@ -244,19 +283,28 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
self.ensure_consumer_group_exists(stream_id, topic_id, group_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().leave_consumer_group(
- session.get_user_id(),
- stream_id,
- topic_id,
- ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - permission denied to leave consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), stream_id, topic_id))?;
- }
+
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+
+ self.permissioner
+ .leave_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - permission denied to leave consumer group for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ stream_id,
+ topic_id
+ )
+ })?;
+
self.leave_consumer_group_base(stream_id, topic_id, group_id, session.client_id)
}
@@ -267,46 +315,96 @@ impl IggyShard {
group_id: &Identifier,
client_id: u32,
) -> Result<(), IggyError> {
- let Some(_) = self.streams.with_consumer_group_by_id_mut(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::leave_consumer_group(client_id),
- ) else {
- return Err(IggyError::ConsumerGroupMemberNotFound(
- client_id,
- group_id.clone(),
- topic_id.clone(),
- ));
- };
+ // Use SharedMetadata for ID resolution
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+ let numeric_group_id = self
+ .shared_metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, group_id)
+ .expect("Consumer group must exist");
- self.streams.with_consumer_group_by_id_mut(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::rebalance_consumer_group(),
+ // Update SharedMetadata (source of truth) - thread-safe via rcu()
+ // Returns the member_id if found
+ let member_id = self.shared_metadata.leave_consumer_group(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
+ client_id,
);
- // Update ClientManager state
- let stream_id_value = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- let topic_id_value =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let group_id_value = self.streams.with_consumer_group_by_id(
- stream_id,
- topic_id,
- group_id,
- topics::helpers::get_consumer_group_id(),
- );
+ match member_id {
+ Some(member_id) => {
+ // Clean up the member's partition index
+ self.shared_metadata.remove_member_partition_index(
+ numeric_stream_id,
+ numeric_topic_id,
+ numeric_group_id,
+ member_id,
+ );
+ }
+ None => {
+ return Err(IggyError::ConsumerGroupMemberNotFound(
+ client_id,
+ group_id.clone(),
+ topic_id.clone(),
+ ));
+ }
+ }
+
+ // Update ClientManager
+ self.client_manager
+ .leave_consumer_group(client_id, numeric_stream_id, numeric_topic_id, numeric_group_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}",
+ client_id
+ )
+ })?;
- self.client_manager.leave_consumer_group(
- client_id,
- stream_id_value,
- topic_id_value,
- group_id_value,
- ).error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}", client_id))?;
Ok(())
}
+
+ /// Get consumer group response from SharedMetadata.
+ /// Works on all shards - SharedMetadata is the source of truth.
+ pub fn get_consumer_group_from_shared_metadata(
+ &self,
+ stream_id: usize,
+ topic_id: usize,
+ group_id: usize,
+ ) -> bytes::Bytes {
+ use bytes::{BufMut, BytesMut};
+
+ let metadata = self.shared_metadata.load();
+ let key = (stream_id, topic_id, group_id);
+
+ let Some(cg_meta) = metadata.consumer_groups.get(&key) else {
+ return bytes::Bytes::new();
+ };
+
+ let mut bytes = BytesMut::new();
+
+ // Header: id, partitions_count, members_count, name_len, name
+ bytes.put_u32_le(cg_meta.id as u32);
+ bytes.put_u32_le(cg_meta.partitions.len() as u32);
+ bytes.put_u32_le(cg_meta.members.len() as u32);
+ bytes.put_u8(cg_meta.name.len() as u8);
+ bytes.put_slice(cg_meta.name.as_bytes());
+
+ // Serialize members
+ for member in cg_meta.members.iter() {
+ bytes.put_u32_le(member.id as u32);
+ bytes.put_u32_le(member.partitions.len() as u32);
+ for &partition_id in &member.partitions {
+ bytes.put_u32_le(partition_id as u32);
+ }
+ }
+
+ bytes.freeze()
+ }
}
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index 74cdb82f1..7d8d03dd5 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -1,34 +1,33 @@
/* Licensed to the Apache Software Foundation (ASF) under one
- polling_consumer: &PollingConsumer,
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
use super::COMPONENT;
use crate::{
shard::IggyShard,
streaming::{
- partitions,
+ partitions::consumer_offset::ConsumerOffset,
polling_consumer::{ConsumerGroupId, PollingConsumer},
session::Session,
- streams, topics,
},
};
use err_trail::ErrContext;
-use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
+use iggy_common::{Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError};
+use std::sync::atomic::Ordering;
impl IggyShard {
pub async fn store_consumer_offset(
@@ -42,24 +41,22 @@ impl IggyShard {
) -> Result<(PollingConsumer, usize), IggyError> {
self.ensure_authenticated(session)?;
self.ensure_topic_exists(stream_id, topic_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().store_consumer_offset(
- session.get_user_id(),
- stream_id,
- topic_id
- ).error(|e: &IggyError| {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+ self.permissioner
+ .store_consumer_offset(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
format!(
- "{COMPONENT} (error: {e}) - permission denied to store consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}",
+ "{COMPONENT} (error: {e}) - permission denied to store consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {numeric_topic_id} and stream with ID: {numeric_stream_id}",
session.get_user_id(),
)
})?;
- }
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
stream_id,
topic_id,
@@ -74,14 +71,19 @@ impl IggyShard {
self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
self.store_consumer_offset_base(
- stream_id,
- topic_id,
+ numeric_stream_id,
+ numeric_topic_id,
&polling_consumer,
partition_id,
offset,
);
- self.persist_consumer_offset_to_disk(stream_id, topic_id, &polling_consumer, partition_id)
- .await?;
+ self.persist_consumer_offset_to_disk(
+ numeric_stream_id,
+ numeric_topic_id,
+ &polling_consumer,
+ partition_id,
+ )
+ .await?;
Ok((polling_consumer, partition_id))
}
@@ -95,24 +97,22 @@ impl IggyShard {
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
self.ensure_authenticated(session)?;
self.ensure_topic_exists(stream_id, topic_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().get_consumer_offset(
- session.get_user_id(),
- stream_id,
- topic_id
- ).error(|e: &IggyError| {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+ self.permissioner
+ .get_consumer_offset(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
format!(
- "{COMPONENT} (error: {e}) - permission denied to get consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}",
+ "{COMPONENT} (error: {e}) - permission denied to get consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {numeric_topic_id} and stream with ID: {numeric_stream_id}",
session.get_user_id()
)
})?;
- }
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
stream_id,
topic_id,
@@ -126,20 +126,50 @@ impl IggyShard {
};
self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
+ // Get the partition's current offset from stats (messages_count - 1, or 0 if empty)
+ use iggy_common::sharding::IggyNamespace;
+ let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let partition_current_offset = self
+ .shared_metadata
+ .get_partition_stats(&ns)
+ .map(|s| {
+ let count = s.messages_count_inconsistent();
+ if count > 0 { count - 1 } else { 0 }
+ })
+ .unwrap_or(0);
+
let offset = match polling_consumer {
- PollingConsumer::Consumer(id, _) => self.streams.with_partition_by_id(
- stream_id,
- topic_id,
- partition_id,
- partitions::helpers::get_consumer_offset(id),
- ),
+ PollingConsumer::Consumer(id, _) => {
+ let offsets = self.shared_consumer_offsets.get_consumer_offsets(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ );
+ offsets.and_then(|co| {
+ let guard = co.pin();
+ guard.get(&id).map(|item| ConsumerOffsetInfo {
+ partition_id: partition_id as u32,
+ current_offset: partition_current_offset,
+ stored_offset: item.offset.load(Ordering::Relaxed),
+ })
+ })
+ }
PollingConsumer::ConsumerGroup(consumer_group_id, _) => {
- self.streams.with_partition_by_id(
- stream_id,
- topic_id,
+ let offsets = self.shared_consumer_offsets.get_consumer_group_offsets(
+ numeric_stream_id,
+ numeric_topic_id,
partition_id,
- partitions::helpers::get_consumer_group_offset(consumer_group_id),
- )
+ );
+ offsets.and_then(|co| {
+ let guard = co.pin();
+ guard
+ .get(&consumer_group_id)
+ .map(|item| ConsumerOffsetInfo {
+ partition_id: partition_id as u32,
+ current_offset: partition_current_offset,
+ stored_offset: item.offset.load(Ordering::Relaxed),
+ })
+ })
}
};
Ok(offset)
@@ -155,24 +185,22 @@ impl IggyShard {
) -> Result<(PollingConsumer, usize), IggyError> {
self.ensure_authenticated(session)?;
self.ensure_topic_exists(stream_id, topic_id)?;
- {
- let topic_id =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
- let stream_id = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- self.permissioner.borrow().delete_consumer_offset(
- session.get_user_id(),
- stream_id,
- topic_id
- ).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - permission denied to delete consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {topic_id} and stream with ID: {stream_id}",
- session.get_user_id(),
- )
- })?;
- }
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+ self.permissioner
+ .delete_consumer_offset(session.get_user_id(), numeric_stream_id, numeric_topic_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - permission denied to delete consumer offset for user with ID: {}, consumer: {consumer} in topic with ID: {numeric_topic_id} and stream with ID: {numeric_stream_id}",
+ session.get_user_id(),
+ )
+ })?;
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
stream_id,
topic_id,
@@ -186,8 +214,12 @@ impl IggyShard {
};
self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
- let path =
- self.delete_consumer_offset_base(stream_id, topic_id, &polling_consumer, partition_id)?;
+ let path = self.delete_consumer_offset_base(
+ numeric_stream_id,
+ numeric_topic_id,
+ &polling_consumer,
+ partition_id,
+ )?;
self.delete_consumer_offset_from_disk(&path).await?;
Ok((polling_consumer, partition_id))
}
@@ -199,36 +231,42 @@ impl IggyShard {
topic_id: &Identifier,
partition_ids: &[usize],
) -> Result<(), IggyError> {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+
for &partition_id in partition_ids {
- // Skip if offset does not exist.
- let has_offset = self
- .streams
- .with_partition_by_id(
- stream_id,
- topic_id,
- partition_id,
- partitions::helpers::get_consumer_group_offset(cg_id),
- )
- .is_some();
- if !has_offset {
+ let offsets = self.shared_consumer_offsets.get_consumer_group_offsets(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ );
+
+ let Some(offsets) = offsets else {
continue;
- }
+ };
- let path = self.streams
- .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_group_offset(cg_id))
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group offset for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}",
- cg_id, partition_id, topic_id, stream_id
- )
- })?;
+ let path = {
+ let guard = offsets.pin();
+ guard.get(&cg_id).map(|item| item.path.clone())
+ };
- self.delete_consumer_offset_from_disk(&path).await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}",
- cg_id, partition_id, topic_id, stream_id
- )
- })?;
+ if let Some(path) = path {
+ offsets.pin().remove(&cg_id);
+ self.delete_consumer_offset_from_disk(&path)
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}",
+ cg_id, partition_id, topic_id, stream_id
+ )
+ })?;
+ }
}
Ok(())
@@ -236,128 +274,145 @@ impl IggyShard {
fn store_consumer_offset_base(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ stream_id: usize,
+ topic_id: usize,
polling_consumer: &PollingConsumer,
partition_id: usize,
offset: u64,
) {
- let stream_id_num = self
- .streams
- .with_stream_by_id(stream_id, streams::helpers::get_stream_id());
- let topic_id_num =
- self.streams
- .with_topic_by_id(stream_id, topic_id, topics::helpers::get_topic_id());
-
match polling_consumer {
PollingConsumer::Consumer(id, _) => {
- self.streams.with_partition_by_id(
+ let offsets = self.shared_consumer_offsets.get_consumer_offsets(
stream_id,
topic_id,
partition_id,
- partitions::helpers::store_consumer_offset(
- *id,
- stream_id_num,
- topic_id_num,
- partition_id,
- offset,
- &self.config.system,
- ),
);
+
+ if let Some(offsets) = offsets {
+ let guard = offsets.pin();
+ if let Some(existing) = guard.get(id) {
+ existing.offset.store(offset, Ordering::Relaxed);
+ } else {
+ let dir_path = self.config.system.get_consumer_offsets_path(
+ stream_id,
+ topic_id,
+ partition_id,
+ );
+ let path = format!("{}/{}", dir_path, id);
+ let consumer_offset =
+ ConsumerOffset::new(ConsumerKind::Consumer, *id as u32, offset, path);
+ drop(guard);
+ offsets.pin().insert(*id, consumer_offset);
+ }
+ }
}
- PollingConsumer::ConsumerGroup(consumer_group_id, _) => {
- self.streams.with_partition_by_id(
+ PollingConsumer::ConsumerGroup(cg_id, _) => {
+ let offsets = self.shared_consumer_offsets.get_consumer_group_offsets(
stream_id,
topic_id,
partition_id,
- partitions::helpers::store_consumer_group_offset(
- *consumer_group_id,
- stream_id_num,
- topic_id_num,
- partition_id,
- offset,
- &self.config.system,
- ),
);
+
+ if let Some(offsets) = offsets {
+ let guard = offsets.pin();
+ if let Some(existing) = guard.get(cg_id) {
+ existing.offset.store(offset, Ordering::Relaxed);
+ } else {
+ let dir_path = self.config.system.get_consumer_group_offsets_path(
+ stream_id,
+ topic_id,
+ partition_id,
+ );
+ let path = format!("{}/{}", dir_path, cg_id.0);
+ let consumer_offset = ConsumerOffset::new(
+ ConsumerKind::ConsumerGroup,
+ cg_id.0 as u32,
+ offset,
+ path,
+ );
+ drop(guard);
+ offsets.pin().insert(*cg_id, consumer_offset);
+ }
+ }
}
}
}
fn delete_consumer_offset_base(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ stream_id: usize,
+ topic_id: usize,
polling_consumer: &PollingConsumer,
partition_id: usize,
) -> Result<String, IggyError> {
match polling_consumer {
PollingConsumer::Consumer(id, _) => {
- self.streams
- .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_offset(*id)).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer offset for consumer with ID: {id} in topic with ID: {topic_id} and stream with ID: {stream_id}",
- )
- })
+ let offsets = self
+ .shared_consumer_offsets
+ .get_consumer_offsets(stream_id, topic_id, partition_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?;
+
+ let guard = offsets.pin();
+ let offset = guard
+ .remove(id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?;
+ Ok(offset.path.clone())
}
- PollingConsumer::ConsumerGroup(consumer_group_id, _) => {
- self.streams
- .with_partition_by_id(stream_id, topic_id, partition_id, partitions::helpers::delete_consumer_group_offset(*consumer_group_id)).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group offset for group with ID: {consumer_group_id:?} in topic with ID: {topic_id} and stream with ID: {stream_id}",
- )
- })
+ PollingConsumer::ConsumerGroup(cg_id, _) => {
+ let offsets = self
+ .shared_consumer_offsets
+ .get_consumer_group_offsets(stream_id, topic_id, partition_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?;
+
+ let guard = offsets.pin();
+ let offset = guard
+ .remove(cg_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?;
+ Ok(offset.path.clone())
}
}
}
async fn persist_consumer_offset_to_disk(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ stream_id: usize,
+ topic_id: usize,
polling_consumer: &PollingConsumer,
partition_id: usize,
) -> Result<(), IggyError> {
- match polling_consumer {
+ use crate::streaming::partitions::storage::persist_offset;
+
+ let (offset_value, path) = match polling_consumer {
PollingConsumer::Consumer(id, _) => {
- let (offset_value, path) = self.streams.with_partition_by_id(
- stream_id,
- topic_id,
- partition_id,
- |(.., offsets, _, _)| {
- let hdl = offsets.pin();
- let item = hdl
- .get(id)
- .expect("persist_consumer_offset_to_disk: offset not found");
- let offset = item.offset.load(std::sync::atomic::Ordering::Relaxed);
- let path = item.path.clone();
- (offset, path)
- },
- );
- partitions::storage::persist_offset(&path, offset_value).await
+ let offsets = self
+ .shared_consumer_offsets
+ .get_consumer_offsets(stream_id, topic_id, partition_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?;
+
+ let guard = offsets.pin();
+ let item = guard
+ .get(id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?;
+ (item.offset.load(Ordering::Relaxed), item.path.clone())
}
- PollingConsumer::ConsumerGroup(consumer_group_id, _) => {
- let (offset_value, path) = self.streams.with_partition_by_id(
- stream_id,
- topic_id,
- partition_id,
- move |(.., offsets, _)| {
- let hdl = offsets.pin();
- let item = hdl
- .get(consumer_group_id)
- .expect("persist_consumer_offset_to_disk: offset not found");
- (
- item.offset.load(std::sync::atomic::Ordering::Relaxed),
- item.path.clone(),
- )
- },
- );
- partitions::storage::persist_offset(&path, offset_value).await
+ PollingConsumer::ConsumerGroup(cg_id, _) => {
+ let offsets = self
+ .shared_consumer_offsets
+ .get_consumer_group_offsets(stream_id, topic_id, partition_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?;
+
+ let guard = offsets.pin();
+ let item = guard
+ .get(cg_id)
+ .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?;
+ (item.offset.load(Ordering::Relaxed), item.path.clone())
}
- }
+ };
+ persist_offset(&path, offset_value).await
}
pub async fn delete_consumer_offset_from_disk(&self, path: &str) -> Result<(), IggyError> {
- partitions::storage::delete_persisted_offset(path).await
+ crate::streaming::partitions::storage::delete_persisted_offset(path).await
}
pub fn store_consumer_offset_bypass_auth(
@@ -368,9 +423,18 @@ impl IggyShard {
partition_id: usize,
offset: u64,
) {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+
self.store_consumer_offset_base(
- stream_id,
- topic_id,
+ numeric_stream_id,
+ numeric_topic_id,
polling_consumer,
partition_id,
offset,
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 02e8d9d0e..3e6303106 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -19,14 +19,13 @@
use super::COMPONENT;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
-use crate::shard::namespace::IggyFullNamespace;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
+use crate::streaming::partitions::journal::Journal;
+use crate::streaming::polling_consumer::PollingConsumer;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
-use crate::streaming::traits::MainOps;
-use crate::streaming::{partitions, streams, topics};
use err_trail::ErrContext;
use iggy_common::PooledBuffer;
use iggy_common::sharding::IggyNamespace;
@@ -48,17 +47,19 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_topic_exists(&stream_id, &topic_id)?;
+ // Use shared_metadata for ID resolution to ensure consistency with partition_store keys
let numeric_stream_id = self
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist");
- let numeric_topic_id =
- self.streams
- .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id());
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist");
// Validate permissions for given user on stream and topic.
self.permissioner
- .borrow()
.append_messages(
user_id,
numeric_stream_id,
@@ -85,20 +86,68 @@ impl IggyShard {
{
ShardSendRequestResult::Recoil(message) => {
if let ShardMessage::Request(ShardRequest {
- stream_id,
- topic_id,
+ stream_id: _,
+ topic_id: _,
partition_id,
payload,
}) = message
&& let ShardRequestPayload::SendMessages { batch } = payload
{
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
// Encrypt messages if encryptor is enabled in configuration.
let batch = self.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
- self.streams
- .append_messages(&self.config.system, &self.task_registry, &ns, batch)
+
+ // Use partition_store - initialize on-demand if not yet ready
+ let namespace =
+ IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ // Get partition's metadata from SharedMetadata (source of truth)
+ let metadata = self.shared_metadata.load();
+ let partition_meta = metadata.partitions.get(&namespace);
+
+ // Check if partition_store entry exists AND is valid (not stale)
+ // A partition_store entry is stale if its creation_version doesn't match
+ let needs_init = {
+ let store = self.partition_store.borrow();
+ match (store.get(&namespace), partition_meta) {
+ (Some(data), Some(meta))
+ if data.creation_version == meta.creation_version =>
+ {
+ // Entry exists and is current
+ false
+ }
+ (Some(_), _) => {
+ // Entry exists but is stale (different version or no metadata)
+ drop(store);
+ self.partition_store.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => {
+ // No entry - needs init
+ true
+ }
+ }
+ };
+
+ if needs_init {
+ // Partition not yet initialized (race with CreatedPartitions event)
+ // Initialize it on-demand using SharedMetadata
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(iggy_common::IggyTimestamp::now);
+
+ self.init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ created_at,
+ )
+ .await?;
+ }
+
+ self.append_messages_to_partition_store(&namespace, batch, &self.config.system)
.await?;
+
self.metrics.increment_messages(messages_count as u64);
Ok(())
} else {
@@ -132,15 +181,17 @@ impl IggyShard {
) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> {
self.ensure_topic_exists(&stream_id, &topic_id)?;
+ // Use SharedMetadata for ID resolution (works on all shards)
let numeric_stream_id = self
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
- let numeric_topic_id =
- self.streams
- .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id());
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist after ensure_topic_exists");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist after ensure_topic_exists");
self.permissioner
- .borrow()
.poll_messages(user_id, numeric_stream_id, numeric_topic_id)
.error(|e: &IggyError| format!(
"{COMPONENT} (error: {e}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
@@ -164,22 +215,23 @@ impl IggyShard {
self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
- let current_offset = self.streams.with_partition_by_id(
- &stream_id,
- &topic_id,
- partition_id,
- |(_, _, _, offset, ..)| offset.load(Ordering::Relaxed),
- );
- if args.strategy.kind == PollingKind::Offset && args.strategy.value > current_offset
- || args.count == 0
- {
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ // Early return if count is 0
+ if args.count == 0 {
+ let current_offset = self
+ .partition_store
+ .borrow()
+ .get(&namespace)
+ .map(|data| data.offset.load(Ordering::Relaxed))
+ .unwrap_or(0);
return Ok((
IggyPollMetadata::new(partition_id as u32, current_offset),
IggyMessagesBatchSet::empty(),
));
}
- let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ // Offset validation is done by the owning shard after routing
let payload = ShardRequestPayload::PollMessages { consumer, args };
let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
let message = ShardMessage::Request(request);
@@ -195,27 +247,55 @@ impl IggyShard {
}) = message
&& let ShardRequestPayload::PollMessages { consumer, args } = payload
{
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
+ // Check for stale partition_store entries before polling
+ let metadata = self.shared_metadata.load();
+ let partition_meta = metadata.partitions.get(&namespace);
+
+ let needs_init = {
+ let store = self.partition_store.borrow();
+ match (store.get(&namespace), partition_meta) {
+ (Some(data), Some(meta))
+ if data.creation_version == meta.creation_version =>
+ {
+ false
+ }
+ (Some(_), _) => {
+ drop(store);
+ self.partition_store.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => true,
+ }
+ };
+
+ if needs_init {
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(iggy_common::IggyTimestamp::now);
+ self.init_partition_directly(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_id,
+ created_at,
+ )
+ .await?;
+ }
+
let auto_commit = args.auto_commit;
- let (metadata, batches) =
- self.streams.poll_messages(&ns, consumer, args).await?;
- let stream_id = ns.stream_id();
- let topic_id = ns.topic_id();
+
+ // Use partition_store (source of truth)
+ let (metadata, batches) = self
+ .poll_messages_from_partition_store(&namespace, consumer, args)
+ .await?;
if auto_commit && !batches.is_empty() {
let offset = batches
.last_offset()
.expect("Batch set should have at least one batch");
- self.streams
- .auto_commit_consumer_offset(
- &self.config.system,
- stream_id,
- topic_id,
- partition_id,
- consumer,
- offset,
- )
- .await?;
+ self.auto_commit_consumer_offset_from_partition_store(
+ &namespace, consumer, offset,
+ )
+ .await?;
}
Ok((metadata, batches))
} else {
@@ -252,24 +332,23 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
+ // Use SharedMetadata for ID resolution
let numeric_stream_id = self
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
-
- let numeric_topic_id =
- self.streams
- .with_topic_by_id(&stream_id, &topic_id, topics::helpers::get_topic_id());
+ .shared_metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream must exist after ensure_partition_exists");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic must exist after ensure_partition_exists");
// Validate permissions for given user on stream and topic.
self.permissioner
- .borrow()
.append_messages(user_id, numeric_stream_id, numeric_topic_id)
.error(|e: &IggyError| {
format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, numeric_stream_id as u32, numeric_topic_id as u32)
})?;
- self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
-
let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync };
let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
@@ -280,14 +359,15 @@ impl IggyShard {
{
ShardSendRequestResult::Recoil(message) => {
if let ShardMessage::Request(ShardRequest {
- stream_id,
- topic_id,
partition_id,
payload,
+ ..
}) = message
&& let ShardRequestPayload::FlushUnsavedBuffer { fsync } = payload
{
- self.flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync)
+ let namespace =
+ IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ self.flush_unsaved_buffer_from_partition_store(&namespace, fsync)
.await?;
Ok(())
} else {
@@ -315,33 +395,717 @@ impl IggyShard {
partition_id: usize,
... 10081 lines suppressed ...