You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/02/03 10:50:30 UTC
(iggy) 01/01: refactor(server): streamline command handlers with centralized execution
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch command-handler-impr
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 6d109438e6b92137544081e4c655dbf55b23a38e
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Wed Jan 28 11:38:21 2026 +0100
refactor(server): streamline command handlers with centralized execution
Handlers had duplicated validation and resolution logic scattered across
bypass_auth variants and inline code. Entity writes occurred on any shard,
requiring distributed filesystem locks that added complexity and potential
for race conditions.
Centralizing execution in shard/execution.rs with pre-resolved entity types
(ResolvedStream, ResolvedTopic, etc.) eliminates duplicate code paths and
ensures entity file writes happen exclusively on shard0, removing the need
for fs_locks entirely.
---
core/binary_protocol/src/utils/mapper.rs | 3 +-
.../create_consumer_group_handler.rs | 105 +--
.../delete_consumer_group_handler.rs | 118 +---
.../consumer_groups/get_consumer_group_handler.rs | 27 +-
.../consumer_groups/get_consumer_groups_handler.rs | 37 +-
.../consumer_groups/join_consumer_group_handler.rs | 66 +-
.../leave_consumer_group_handler.rs | 69 +-
.../delete_consumer_offset_handler.rs | 17 +-
.../get_consumer_offset_handler.rs | 30 +-
.../store_consumer_offset_handler.rs | 12 +-
.../messages/flush_unsaved_buffer_handler.rs | 22 +-
.../handlers/messages/poll_messages_handler.rs | 11 +-
.../handlers/messages/send_messages_handler.rs | 39 +-
.../partitions/create_partitions_handler.rs | 102 +--
.../partitions/delete_partitions_handler.rs | 104 +--
.../create_personal_access_token_handler.rs | 90 +--
.../delete_personal_access_token_handler.rs | 74 +--
.../handlers/segments/delete_segments_handler.rs | 78 +--
.../handlers/streams/create_stream_handler.rs | 76 +--
.../handlers/streams/delete_stream_handler.rs | 72 +--
.../binary/handlers/streams/get_stream_handler.rs | 25 +-
.../binary/handlers/streams/get_streams_handler.rs | 15 +-
.../handlers/streams/purge_stream_handler.rs | 77 +--
.../handlers/streams/update_stream_handler.rs | 69 +-
.../binary/handlers/system/get_stats_handler.rs | 45 +-
.../binary/handlers/topics/create_topic_handler.rs | 129 +---
.../binary/handlers/topics/delete_topic_handler.rs | 99 +--
.../binary/handlers/topics/get_topic_handler.rs | 24 +-
.../binary/handlers/topics/get_topics_handler.rs | 12 +-
.../binary/handlers/topics/purge_topic_handler.rs | 87 +--
.../binary/handlers/topics/update_topic_handler.rs | 102 +--
.../handlers/users/change_password_handler.rs | 97 +--
.../binary/handlers/users/create_user_handler.rs | 100 +--
.../binary/handlers/users/delete_user_handler.rs | 72 +--
.../src/binary/handlers/users/get_user_handler.rs | 17 +-
.../src/binary/handlers/users/get_users_handler.rs | 8 +-
.../handlers/users/update_permissions_handler.rs | 86 +--
.../binary/handlers/users/update_user_handler.rs | 96 +--
core/server/src/binary/mapper.rs | 215 ++++++-
core/server/src/http/consumer_groups.rs | 186 ++----
core/server/src/http/consumer_offsets.rs | 40 +-
core/server/src/http/http_shard_wrapper.rs | 251 ++------
core/server/src/http/messages.rs | 20 +-
core/server/src/http/partitions.rs | 130 +---
core/server/src/http/personal_access_tokens.rs | 74 +--
core/server/src/http/streams.rs | 207 ++----
core/server/src/http/topics.rs | 311 ++-------
core/server/src/http/users.rs | 246 ++-----
core/server/src/io/fs_locks.rs | 51 --
core/server/src/io/mod.rs | 1 -
core/server/src/metadata/reader.rs | 583 ++++++++++++++++-
core/server/src/shard/builder.rs | 1 -
core/server/src/shard/communication.rs | 83 ++-
core/server/src/shard/execution.rs | 705 +++++++++++++++++++++
core/server/src/shard/handlers.rs | 684 ++++----------------
core/server/src/shard/mod.rs | 37 +-
core/server/src/shard/system/clients.rs | 40 +-
core/server/src/shard/system/consumer_groups.rs | 117 ++--
core/server/src/shard/system/consumer_offsets.rs | 124 ++--
core/server/src/shard/system/messages.rs | 265 +++-----
core/server/src/shard/system/partitions.rs | 37 +-
.../src/shard/system/personal_access_tokens.rs | 28 +-
core/server/src/shard/system/segments.rs | 2 +-
core/server/src/shard/system/streams.rs | 227 ++-----
core/server/src/shard/system/topics.rs | 281 ++------
core/server/src/shard/system/users.rs | 36 --
core/server/src/shard/system/utils.rs | 327 +++++++---
core/server/src/shard/transmission/frame.rs | 46 +-
core/server/src/shard/transmission/message.rs | 237 +++----
69 files changed, 3146 insertions(+), 4758 deletions(-)
diff --git a/core/binary_protocol/src/utils/mapper.rs b/core/binary_protocol/src/utils/mapper.rs
index 356d65931..ade37593a 100644
--- a/core/binary_protocol/src/utils/mapper.rs
+++ b/core/binary_protocol/src/utils/mapper.rs
@@ -656,8 +656,7 @@ pub fn map_topic(payload: Bytes) -> Result<TopicDetails, IggyError> {
compression_algorithm: topic.compression_algorithm,
max_topic_size: topic.max_topic_size,
replication_factor: topic.replication_factor,
- #[allow(clippy::cast_possible_truncation)]
- partitions_count: partitions.len() as u32,
+ partitions_count: topic.partitions_count,
partitions,
};
Ok(topic)
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 7eef62841..2945bb41a 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -19,20 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateConsumerGroupWithId;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::create_consumer_group::CreateConsumerGroup;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -51,94 +45,23 @@ impl ServerCommandHandler for CreateConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_create_consumer_group(session.get_user_id(), stream_id, topic_id)?;
- let request = ShardRequest {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partition_id: 0,
- payload: ShardRequestPayload::CreateConsumerGroup {
+ let request =
+ ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest {
user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- name: self.name.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreateConsumerGroup {
- stream_id: recoil_stream_id,
- topic_id: recoil_topic_id,
- name,
- ..
- } = payload
- {
- let cg_id =
- shard.create_consumer_group(&recoil_stream_id, &recoil_topic_id, name)?;
-
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
- group_id: cg_id as u32,
- command: self,
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create consumer group for stream_id: {stream_id}, topic_id: {topic_id}, group_id: {cg_id}, session: {session}"
- )
- })?;
+ command: self,
+ });
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
-
- let cg_identifier = Identifier::numeric(cg_id as u32).unwrap();
- let response = shard
- .metadata
- .get_consumer_group(numeric_stream_id, numeric_topic_id, cg_id)
- .map(|cg| mapper::map_consumer_group_from_meta(&cg))
- .ok_or_else(|| {
- IggyError::ConsumerGroupIdNotFound(cg_identifier, topic_id.clone())
- })?;
- sender.send_ok_response(&response).await?;
- } else {
- unreachable!(
- "Expected a CreateConsumerGroup request inside of CreateConsumerGroup handler"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateConsumerGroupResponse(data) => {
+ sender
+ .send_ok_response(&mapper::map_consumer_group_from_response(&data))
+ .await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateConsumerGroupResponse(cg_id) => {
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreateConsumerGroupResponse"),
+ }
- let cg_identifier = Identifier::numeric(cg_id as u32).unwrap();
- let response = shard
- .metadata
- .get_consumer_group(numeric_stream_id, numeric_topic_id, cg_id)
- .map(|cg| mapper::map_consumer_group_from_meta(&cg))
- .ok_or_else(|| {
- IggyError::ConsumerGroupIdNotFound(cg_identifier, self.topic_id.clone())
- })?;
- sender.send_ok_response(&response).await?;
- }
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a CreateConsumerGroupResponse inside of CreateConsumerGroup handler"
- ),
- },
- };
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
index 8bc03be43..ffb35d0ec 100644
--- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
@@ -19,18 +19,11 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::metadata::ConsumerGroupMeta;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
-use crate::streaming::polling_consumer::ConsumerGroupId;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_consumer_group::DeleteConsumerGroup;
use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
@@ -51,112 +44,21 @@ impl ServerCommandHandler for DeleteConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard.metadata.perm_delete_consumer_group(
- session.get_user_id(),
- numeric_stream_id,
- numeric_topic_id,
- )?;
- let request = ShardRequest {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partition_id: 0,
- payload: ShardRequestPayload::DeleteConsumerGroup {
+ let request =
+ ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest {
user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- group_id: self.group_id.clone(),
- },
- };
+ command: self,
+ });
- let message = ShardMessage::Request(request);
- let cg_meta: ConsumerGroupMeta = match shard
- .send_request_to_shard_or_recoil(None, message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::DeleteConsumerGroup {
- stream_id,
- topic_id,
- group_id,
- ..
- } = payload
- {
- shard.delete_consumer_group(&stream_id, &topic_id, &group_id).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {} for topic with ID: {} in stream with ID: {} for session: {}",
- group_id, topic_id, stream_id, session
- )
- })?
- } else {
- unreachable!(
- "Expected a DeleteConsumerGroup request inside of DeleteConsumerGroup handler"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteConsumerGroupResponse => {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- }
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a DeleteConsumerGroupResponse inside of DeleteConsumerGroup handler"
- ),
- },
- };
-
- let cg_id = cg_meta.id;
-
- for (_, member) in cg_meta.members.iter() {
- if let Err(err) = shard.client_manager.leave_consumer_group(
- member.client_id,
- numeric_stream_id,
- numeric_topic_id,
- cg_id,
- ) {
- tracing::warn!(
- "{COMPONENT} (error: {err}) - failed to make client leave consumer group for client ID: {}, group ID: {}",
- member.client_id,
- cg_id
- );
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteConsumerGroupResponse => {
+ sender.send_empty_ok_response().await?;
}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeleteConsumerGroupResponse"),
}
- let cg_id_spez = ConsumerGroupId(cg_id);
- shard.delete_consumer_group_offsets(
- cg_id_spez,
- &self.stream_id,
- &self.topic_id,
- &cg_meta.partitions,
- ).await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
- cg_id_spez,
- self.stream_id,
- self.topic_id
- )
- })?;
-
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::DeleteConsumerGroup(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete consumer group for stream_id: {}, topic_id: {}, group_id: {cg_id}, session: {session}",
- stream_id, topic_id
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
index ac716aabc..c0eddb8af 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
@@ -43,29 +43,20 @@ impl ServerCommandHandler for GetConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Ok((stream_id, topic_id, numeric_group_id)) =
- shard.resolve_consumer_group_id(&self.stream_id, &self.topic_id, &self.group_id)
+
+ let Some(consumer_group) = shard.metadata.query_consumer_group(
+ session.get_user_id(),
+ &self.stream_id,
+ &self.topic_id,
+ &self.group_id,
+ )?
else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
};
- if shard
- .metadata
- .perm_get_consumer_group(session.get_user_id(), stream_id, topic_id)
- .is_err()
- {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- }
- let consumer_group = shard
- .metadata
- .get_consumer_group(stream_id, topic_id, numeric_group_id)
- .map(|cg| mapper::map_consumer_group_from_meta(&cg))
- .ok_or_else(|| {
- IggyError::ConsumerGroupIdNotFound(self.group_id.clone(), self.topic_id.clone())
- })?;
- sender.send_ok_response(&consumer_group).await?;
+ let response = mapper::map_consumer_group_from_meta(&consumer_group);
+ sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
index 16b606092..f798605c5 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
@@ -20,9 +20,10 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use bytes::{BufMut, BytesMut};
+use bytes::Bytes;
use iggy_common::get_consumer_groups::GetConsumerGroups;
use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
@@ -42,29 +43,19 @@ impl ServerCommandHandler for GetConsumerGroups {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_get_consumer_groups(session.get_user_id(), stream_id, topic_id)?;
- let consumer_groups = shard.metadata.with_metadata(|m| {
- m.streams
- .get(stream_id)
- .and_then(|s| s.topics.get(topic_id))
- .map(|topic| {
- let mut bytes = BytesMut::new();
- for (_, cg_meta) in topic.consumer_groups.iter() {
- bytes.put_u32_le(cg_meta.id as u32);
- bytes.put_u32_le(cg_meta.partitions.len() as u32);
- bytes.put_u32_le(cg_meta.members.len() as u32);
- bytes.put_u8(cg_meta.name.len() as u8);
- bytes.put_slice(cg_meta.name.as_bytes());
- }
- bytes.freeze()
- })
- .unwrap_or_default()
- });
- sender.send_ok_response(&consumer_groups).await?;
+ let Some(consumer_groups) = shard.metadata.query_consumer_groups(
+ session.get_user_id(),
+ &self.stream_id,
+ &self.topic_id,
+ )?
+ else {
+ sender.send_ok_response(&Bytes::new()).await?;
+ return Ok(HandlerResult::Finished);
+ };
+
+ let response = mapper::map_consumer_groups(&consumer_groups);
+ sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
index 6a593dd4d..18b220ad1 100644
--- a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
@@ -19,15 +19,11 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::join_consumer_group::JoinConsumerGroup;
use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
@@ -48,61 +44,21 @@ impl ServerCommandHandler for JoinConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_join_consumer_group(session.get_user_id(), stream_id, topic_id)?;
- shard.ensure_consumer_group_exists(&self.stream_id, &self.topic_id, &self.group_id)?;
- let request = ShardRequest {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partition_id: 0,
- payload: ShardRequestPayload::JoinConsumerGroup {
- user_id: session.get_user_id(),
- client_id: session.client_id,
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- group_id: self.group_id.clone(),
- },
- };
+ let request = ShardRequest::control_plane(ShardRequestPayload::JoinConsumerGroupRequest {
+ user_id: session.get_user_id(),
+ client_id: session.client_id,
+ command: self,
+ });
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::JoinConsumerGroup {
- client_id,
- stream_id,
- topic_id,
- group_id,
- ..
- } = payload
- {
- shard
- .join_consumer_group(client_id, &stream_id, &topic_id, &group_id)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to join consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
- stream_id, topic_id, group_id, session
- )
- })?;
- } else {
- unreachable!(
- "Expected a JoinConsumerGroup request inside of JoinConsumerGroup handler"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::JoinConsumerGroupResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::JoinConsumerGroupResponse => {}
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a JoinConsumerGroupResponse inside of JoinConsumerGroup handler"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected JoinConsumerGroupResponse"),
}
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
index 12eb07b45..3134086d6 100644
--- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
@@ -16,21 +16,16 @@
* under the License.
*/
-use super::COMPONENT;
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
-use iggy_common::IggyError;
-use iggy_common::SenderKind;
use iggy_common::leave_consumer_group::LeaveConsumerGroup;
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -49,61 +44,21 @@ impl ServerCommandHandler for LeaveConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_leave_consumer_group(session.get_user_id(), stream_id, topic_id)?;
- shard.ensure_consumer_group_exists(&self.stream_id, &self.topic_id, &self.group_id)?;
- let request = ShardRequest {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partition_id: 0,
- payload: ShardRequestPayload::LeaveConsumerGroup {
- user_id: session.get_user_id(),
- client_id: session.client_id,
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- group_id: self.group_id.clone(),
- },
- };
+ let request = ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupRequest {
+ user_id: session.get_user_id(),
+ client_id: session.client_id,
+ command: self,
+ });
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::LeaveConsumerGroup {
- client_id,
- stream_id,
- topic_id,
- group_id,
- ..
- } = payload
- {
- shard
- .leave_consumer_group(client_id, &stream_id, &topic_id, &group_id)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to leave consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
- stream_id, topic_id, group_id, session
- )
- })?;
- } else {
- unreachable!(
- "Expected a LeaveConsumerGroup request inside of LeaveConsumerGroup handler"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::LeaveConsumerGroupResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::LeaveConsumerGroupResponse => {}
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a LeaveConsumerGroupResponse inside of LeaveConsumerGroup handler"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected LeaveConsumerGroupResponse"),
}
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
index f8f024915..0d986e921 100644
--- a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
@@ -43,18 +43,13 @@ impl ServerCommandHandler for DeleteConsumerOffset {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
+ let topic = shard.resolve_topic_for_delete_consumer_offset(
+ session.get_user_id(),
+ &self.stream_id,
+ &self.topic_id,
+ )?;
shard
- .metadata
- .perm_delete_consumer_offset(session.get_user_id(), stream_id, topic_id)?;
- shard
- .delete_consumer_offset(
- session.client_id,
- self.consumer,
- &self.stream_id,
- &self.topic_id,
- self.partition_id,
- )
+ .delete_consumer_offset(session.client_id, self.consumer, topic, self.partition_id)
.await
.error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer offset for topic with ID: {} in stream with ID: {} partition ID: {:#?}, session: {}",
self.topic_id, self.stream_id, self.partition_id, session
diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
index 070b54b5e..1e207d78a 100644
--- a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
@@ -22,6 +22,7 @@ use crate::binary::command::{
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::ResolvedTopic;
use crate::streaming::session::Session;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -43,27 +44,24 @@ impl ServerCommandHandler for GetConsumerOffset {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Ok((stream_id, topic_id)) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)
+
+ let Some(resolved) = shard.metadata.resolve_for_consumer_offset(
+ session.get_user_id(),
+ &self.stream_id,
+ &self.topic_id,
+ )?
else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
};
- if shard
- .metadata
- .perm_get_consumer_offset(session.get_user_id(), stream_id, topic_id)
- .is_err()
- {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- }
+
+ let topic = ResolvedTopic {
+ stream_id: resolved.stream_id,
+ topic_id: resolved.topic_id,
+ };
+
let Ok(offset) = shard
- .get_consumer_offset(
- session.client_id,
- self.consumer,
- &self.stream_id,
- &self.topic_id,
- self.partition_id,
- )
+ .get_consumer_offset(session.client_id, self.consumer, topic, self.partition_id)
.await
else {
sender.send_empty_ok_response().await?;
diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
index f28f2113f..a0d2f2cbd 100644
--- a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
@@ -45,16 +45,16 @@ impl ServerCommandHandler for StoreConsumerOffset {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_store_consumer_offset(session.get_user_id(), stream_id, topic_id)?;
+ let topic = shard.resolve_topic_for_store_consumer_offset(
+ session.get_user_id(),
+ &self.stream_id,
+ &self.topic_id,
+ )?;
shard
.store_consumer_offset(
session.client_id,
self.consumer,
- &self.stream_id,
- &self.topic_id,
+ topic,
self.partition_id,
self.offset,
)
diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
index 28100048f..300ecc672 100644
--- a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
+++ b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
@@ -22,6 +22,7 @@ use crate::binary::command::{
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::ResolvedPartition;
use crate::streaming::session::Session;
use err_trail::ErrContext;
use iggy_common::{FlushUnsavedBuffer, IggyError, SenderKind};
@@ -45,24 +46,25 @@ impl ServerCommandHandler for FlushUnsavedBuffer {
shard.ensure_authenticated(session)?;
let user_id = session.get_user_id();
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
+ let stream_id_log = self.stream_id.clone();
+ let topic_id_log = self.topic_id.clone();
let partition_id = self.partition_id;
let fsync = self.fsync;
+ let topic = shard.resolve_topic(&self.stream_id, &self.topic_id)?;
+ let partition = ResolvedPartition {
+ stream_id: topic.stream_id,
+ topic_id: topic.topic_id,
+ partition_id: partition_id as usize,
+ };
+
shard
- .flush_unsaved_buffer(
- user_id,
- self.stream_id,
- self.topic_id,
- partition_id as usize,
- fsync,
- )
+ .flush_unsaved_buffer(user_id, partition, fsync)
.await
.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to flush unsaved buffer for stream_id: {}, topic_id: {}, partition_id: {}, session: {}",
- stream_id, topic_id, partition_id, session
+ stream_id_log, topic_id_log, partition_id, session
)
})?;
sender.send_empty_ok_response().await?;
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index e37bbe284..808fc00a0 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -70,16 +70,9 @@ impl ServerCommandHandler for PollMessages {
let user_id = session.get_user_id();
let client_id = session.client_id;
+ let topic = shard.resolve_topic_for_poll(user_id, &stream_id, &topic_id)?;
let (metadata, mut batch) = shard
- .poll_messages(
- client_id,
- user_id,
- stream_id,
- topic_id,
- consumer,
- partition_id,
- args,
- )
+ .poll_messages(client_id, topic, consumer, partition_id, args)
.await?;
// Collect all chunks first into a Vec to extend their lifetimes.
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 15bad58b9..713d64ab2 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -18,7 +18,7 @@
use crate::binary::command::{BinaryServerCommand, HandlerResult, ServerCommandHandler};
use crate::shard::IggyShard;
-use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
+use crate::shard::transmission::message::{ResolvedPartition, ShardRequest, ShardRequestPayload};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
use crate::streaming::topics;
@@ -107,18 +107,16 @@ impl ServerCommandHandler for SendMessages {
let batch = IggyMessagesBatchMut::from_indexes_and_messages(indexes, messages_buffer);
batch.validate()?;
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard.metadata.perm_append_messages(
+ let topic = shard.resolve_topic_for_append(
session.get_user_id(),
- numeric_stream_id,
- numeric_topic_id,
+ &self.stream_id,
+ &self.topic_id,
)?;
let partition_id = match self.partitioning.kind {
PartitioningKind::Balanced => shard
.metadata
- .get_next_partition_id(numeric_stream_id, numeric_topic_id)
+ .get_next_partition_id(topic.stream_id, topic.topic_id)
.ok_or(IggyError::TopicIdNotFound(
self.stream_id.clone(),
self.topic_id.clone(),
@@ -131,7 +129,7 @@ impl ServerCommandHandler for SendMessages {
PartitioningKind::MessagesKey => {
let partitions_count = shard
.metadata
- .partitions_count(numeric_stream_id, numeric_topic_id);
+ .partitions_count(topic.stream_id, topic.topic_id);
topics::helpers::calculate_partition_id_by_messages_key_hash(
partitions_count,
&self.partitioning.value,
@@ -139,7 +137,7 @@ impl ServerCommandHandler for SendMessages {
}
};
- let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id);
let user_id = session.get_user_id();
let unsupport_socket_transfer = matches!(
self.partitioning.kind,
@@ -167,19 +165,9 @@ impl ServerCommandHandler for SendMessages {
initial_data: batch,
};
- let request = ShardRequest::new(
- self.stream_id.clone(),
- self.topic_id.clone(),
- partition_id,
- payload,
- );
+ let request = ShardRequest::data_plane(namespace, payload);
- let socket_transfer_msg = ShardMessage::Request(request);
-
- if let Err(e) = shard
- .send_request_to_shard_or_recoil(Some(&namespace), socket_transfer_msg)
- .await
- {
+ if let Err(e) = shard.send_to_data_plane(request).await {
error!("transfer socket to another shard failed, drop connection. {e:?}");
return Ok(HandlerResult::Finished);
}
@@ -191,9 +179,12 @@ impl ServerCommandHandler for SendMessages {
}
}
- shard
- .append_messages(user_id, self.stream_id, self.topic_id, partition_id, batch)
- .await?;
+ let partition = ResolvedPartition {
+ stream_id: topic.stream_id,
+ topic_id: topic.topic_id,
+ partition_id,
+ };
+ shard.append_messages(partition, batch).await?;
sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
index f775edf2f..a5113b50a 100644
--- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
@@ -19,19 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::create_partitions::CreatePartitions;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,92 +44,18 @@ impl ServerCommandHandler for CreatePartitions {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id_numeric, topic_id_numeric) =
- shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard.metadata.perm_create_partitions(
- session.get_user_id(),
- stream_id_numeric,
- topic_id_numeric,
- )?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::CreatePartitions {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions_count: self.partitions_count,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreatePartitions {
- stream_id,
- topic_id,
- partitions_count,
- ..
- } = payload
- {
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- // Get numeric IDs BEFORE create (for rebalance operation)
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
-
- let partition_infos = shard
- .create_partitions(&stream_id, &topic_id, partitions_count)
- .await?;
-
- let event = ShardEvent::CreatedPartitions {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- partitions: partition_infos,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let total_partition_count = shard
- .metadata
- .partitions_count(numeric_stream_id, numeric_topic_id)
- as u32;
- shard.writer().rebalance_consumer_groups_for_topic(
- numeric_stream_id,
- numeric_topic_id,
- total_partition_count,
- );
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::CreatePartitions(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {numeric_stream_id}, topic_id: {numeric_topic_id}, session: {session}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a CreatePartitions request inside of CreatePartitions handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreatePartitionsResponse(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreatePartitionsResponse(_partitions) => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a CreatePartitionsResponse inside of CreatePartitions handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreatePartitionsResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
index 5b7fd0b7a..394f8379f 100644
--- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
@@ -19,19 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_partitions::DeletePartitions;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,94 +44,18 @@ impl ServerCommandHandler for DeletePartitions {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id_numeric, topic_id_numeric) =
- shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard.metadata.perm_delete_partitions(
- session.get_user_id(),
- stream_id_numeric,
- topic_id_numeric,
- )?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::DeletePartitions {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions_count: self.partitions_count,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::DeletePartitions {
- stream_id,
- topic_id,
- partitions_count,
- ..
- } = payload
- {
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- // Get numeric IDs BEFORE delete
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
-
- let deleted_partition_ids = shard
- .delete_partitions(&stream_id, &topic_id, partitions_count)
- .await?;
-
- let event = ShardEvent::DeletedPartitions {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- partitions_count,
- partition_ids: deleted_partition_ids,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let remaining_partition_count = shard
- .metadata
- .partitions_count(numeric_stream_id, numeric_topic_id)
- as u32;
- shard.writer().rebalance_consumer_groups_for_topic(
- numeric_stream_id,
- numeric_topic_id,
- remaining_partition_count,
- );
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::DeletePartitions(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {}, topic_id: {}, session: {session}",
- stream_id, topic_id
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a DeletePartitions request inside of DeletePartitions handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletePartitionsResponse(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeletePartitionsResponse(_partition_ids) => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a DeletePartitionsResponse inside of DeletePartitions handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeletePartitionsResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
index e89ac1dff..9f7d31a15 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
@@ -19,20 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
-use crate::state::models::CreatePersonalAccessTokenWithHash;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::create_personal_access_token::CreatePersonalAccessToken;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -51,75 +45,23 @@ impl ServerCommandHandler for CreatePersonalAccessToken {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let user_id = session.get_user_id();
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::CreatePersonalAccessToken {
- user_id,
- name: self.name.clone(),
- expiry: self.expiry,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreatePersonalAccessToken { name, expiry, .. } =
- payload
- {
- let (personal_access_token, token) = shard
- .create_personal_access_token(user_id, &name, expiry)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create personal access token with name: {name}, user: {user_id}"
- )
- })?;
- let bytes = mapper::map_raw_pat(&token);
- let hash = personal_access_token.token.to_string();
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessToken {
+ user_id: session.get_user_id(),
+ name: self.name.clone(),
+ expiry: self.expiry,
+ });
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash {
- command: CreatePersonalAccessToken {
- name: self.name.to_owned(),
- expiry: self.expiry,
- },
- hash,
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create personal access token with name: {}, session: {session}",
- self.name
- )
- })?;
-
- sender.send_ok_response(&bytes).await?;
- } else {
- unreachable!(
- "Expected a CreatePersonalAccessToken request inside of CreatePersonalAccessToken handler"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreatePersonalAccessTokenResponse(_, token) => {
+ sender
+ .send_ok_response(&mapper::map_raw_pat(&token))
+ .await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreatePersonalAccessTokenResponse(_pat, token) => {
- let bytes = mapper::map_raw_pat(&token);
- sender.send_ok_response(&bytes).await?;
- }
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a CreatePersonalAccessTokenResponse inside of CreatePersonalAccessToken handler"
- ),
- },
- };
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreatePersonalAccessTokenResponse"),
+ }
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
index 6c7ebb9a3..7be67056d 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
@@ -19,18 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_personal_access_token::DeletePersonalAccessToken;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -49,65 +44,20 @@ impl ServerCommandHandler for DeletePersonalAccessToken {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let token_name = self.name.clone();
- let user_id = session.get_user_id();
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::DeletePersonalAccessToken {
- user_id,
- name: self.name.clone(),
- },
- };
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::DeletePersonalAccessToken { name, .. } = payload
- {
- shard.delete_personal_access_token(user_id, &name).error(
- |e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete personal access token with name: {name}, user: {user_id}"
- )
- },
- )?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessToken {
+ user_id: session.get_user_id(),
+ name: self.name.clone(),
+ });
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::DeletePersonalAccessToken(DeletePersonalAccessToken {
- name: self.name,
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete personal access token with name: {token_name}, session: {session}"
- )
- })?;
-
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a DeletePersonalAccessToken request inside of DeletePersonalAccessToken handler"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletePersonalAccessTokenResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeletePersonalAccessTokenResponse => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => return Err(err),
- _ => unreachable!(
- "Expected a DeletePersonalAccessTokenResponse inside of DeletePersonalAccessToken handler"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeletePersonalAccessTokenResponse"),
}
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
index f22117844..4a59a2d28 100644
--- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs
+++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
@@ -19,16 +19,11 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_segments::DeleteSegments;
use iggy_common::sharding::IggyNamespace;
use iggy_common::{IggyError, SenderKind};
@@ -51,73 +46,32 @@ impl ServerCommandHandler for DeleteSegments {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
let partition_id = self.partition_id as usize;
let segments_count = self.segments_count;
- let (numeric_stream_id, numeric_topic_id, _) =
- shard.resolve_partition_id(&stream_id, &topic_id, partition_id)?;
- shard.metadata.perm_delete_segments(
+ let partition = shard.resolve_partition_for_delete_segments(
session.get_user_id(),
- numeric_stream_id,
- numeric_topic_id,
+ &self.stream_id,
+ &self.topic_id,
+ partition_id,
)?;
- let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let namespace = IggyNamespace::new(
+ partition.stream_id,
+ partition.topic_id,
+ partition.partition_id,
+ );
let payload = ShardRequestPayload::DeleteSegments { segments_count };
- let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
- let message = ShardMessage::Request(request);
-
- match shard
- .send_request_to_shard_or_recoil(Some(&namespace), message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(crate::shard::transmission::message::ShardRequest {
- stream_id: recoil_stream_id,
- topic_id: recoil_topic_id,
- partition_id: recoil_partition_id,
- payload,
- }) = message
- && let ShardRequestPayload::DeleteSegments { segments_count } = payload
- {
- let (stream, topic) =
- shard.resolve_topic_id(&recoil_stream_id, &recoil_topic_id)?;
- shard
- .delete_segments_base(stream, topic, recoil_partition_id, segments_count)
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete segments for topic with ID: {recoil_topic_id} in stream with ID: {recoil_stream_id}, session: {session}",
- )
- })?;
-
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::DeleteSegments(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply 'delete segments' command for partition with ID: {partition_id} in topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
- )
- })?;
+ let request = ShardRequest::data_plane(namespace, payload);
- sender.send_empty_ok_response().await?;
- } else {
- return Err(IggyError::InvalidCommand);
- }
- }
- ShardSendRequestResult::Response(response) => {
- if !matches!(response, ShardResponse::DeleteSegments) {
- return Err(IggyError::InvalidCommand);
- }
+ match shard.send_to_data_plane(request).await? {
+ ShardResponse::DeleteSegments => {
sender.send_empty_ok_response().await?;
}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeleteSegments"),
}
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs
index 4feacdca6..f1a4e5e66 100644
--- a/core/server/src/binary/handlers/streams/create_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs
@@ -19,19 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateStreamWithId;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::create_stream::CreateStream;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,63 +45,20 @@ impl ServerCommandHandler for CreateStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard.metadata.perm_create_stream(session.get_user_id())?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::CreateStream {
- user_id: session.get_user_id(),
- name: self.name.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreateStream { name, .. } = payload
- {
- // Acquire stream lock to serialize filesystem operations
- let _stream_guard = shard.fs_locks.stream_lock.lock().await;
-
- let created_stream_id = shard.create_stream(name).await?;
-
- let response = shard.get_stream_from_metadata(created_stream_id);
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::CreateStream(CreateStreamWithId {
- stream_id: created_stream_id as u32,
- command: self
- }))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create stream for id: {created_stream_id}, session: {session}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_ok_response(&response).await?;
- } else {
- unreachable!(
- "Expected a CreateStream request inside of CreateStream handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateStreamResponse(data) => {
+ sender
+ .send_ok_response(&mapper::map_stream_from_response(&data))
+ .await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateStreamResponse(created_stream_id) => {
- let response = shard.get_stream_from_metadata(created_stream_id);
- sender.send_ok_response(&response).await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a CreateStreamResponse inside of CreateStream handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreateStreamResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index c41eeca14..0ff0d06ab 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -19,20 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_stream::DeleteStream;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
-use tracing::info;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteStream {
@@ -50,62 +44,18 @@ impl ServerCommandHandler for DeleteStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = shard.resolve_stream_id(&self.stream_id)?;
- shard
- .metadata
- .perm_delete_stream(session.get_user_id(), stream_id)?;
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::DeleteStream {
- user_id: session.get_user_id(),
- stream_id: self.stream_id,
- },
- };
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::DeleteStream { stream_id, .. } = payload
- {
- // Acquire stream lock to serialize filesystem operations
- let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream_info = shard
- .delete_stream(&stream_id)
- .await
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to delete stream with ID: {stream_id}, session: {session}")
- })?;
- info!(
- "Deleted stream with name: {}, ID: {}",
- stream_info.name, stream_info.id
- );
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: stream_id.clone() }))
- .await
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}, session: {session}")
- })?;
- sender.send_empty_ok_response().await?;
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteStreamResponse(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteStreamResponse(_stream_id_num) => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => {
- unreachable!(
- "Expected a DeleteStreamResponse inside of DeleteStream handler, impossible state"
- );
- }
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeleteStreamResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs
index cadaf05b7..1a29a226a 100644
--- a/core/server/src/binary/handlers/streams/get_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs
@@ -20,9 +20,9 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_stream::GetStream;
@@ -44,28 +44,15 @@ impl ServerCommandHandler for GetStream {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- };
-
- let has_permission = shard
+ let Some(stream) = shard
.metadata
- .perm_get_stream(session.get_user_id(), numeric_stream_id)
- .error(|e: &IggyError| {
- format!(
- "permission denied to get stream with ID: {} for user with ID: {}, error: {e}",
- self.stream_id,
- session.get_user_id(),
- )
- })
- .is_ok();
- if !has_permission {
+ .query_stream(session.get_user_id(), &self.stream_id)?
+ else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
- }
+ };
- let response = shard.get_stream_from_metadata(numeric_stream_id);
+ let response = mapper::map_stream(&stream);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs
index 3e2ff4404..8cea11ba7 100644
--- a/core/server/src/binary/handlers/streams/get_streams_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs
@@ -19,11 +19,10 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_streams::GetStreams;
@@ -44,17 +43,9 @@ impl ServerCommandHandler for GetStreams {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard
- .metadata
- .perm_get_streams(session.get_user_id())
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - permission denied to get streams for user {}",
- session.get_user_id()
- )
- })?;
- let response = shard.get_streams_from_metadata();
+ let streams = shard.metadata.query_streams(session.get_user_id())?;
+ let response = mapper::map_streams(&streams);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
index c904bb774..6346fba9e 100644
--- a/core/server/src/binary/handlers/streams/purge_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
@@ -19,19 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::purge_stream::PurgeStream;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,67 +44,18 @@ impl ServerCommandHandler for PurgeStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = shard.resolve_stream_id(&self.stream_id)?;
- shard
- .metadata
- .perm_purge_stream(session.get_user_id(), stream_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::PurgeStream {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- },
- };
-
- let stream_id_for_log = self.stream_id.clone();
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::PurgeStream { stream_id, .. } = payload
- {
- shard.purge_stream(&stream_id).await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to purge stream with id: {stream_id_for_log}, session: {session}"
- )
- })?;
-
- let event = ShardEvent::PurgedStream {
- stream_id: stream_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::PurgeStream(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply purge stream with id: {stream_id_for_log}, session: {session}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a PurgeStream request inside of PurgeStream handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::PurgeStreamResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::PurgeStreamResponse => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a PurgeStreamResponse inside of PurgeStream handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected PurgeStreamResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs
index 332045ebf..2deb7fcd2 100644
--- a/core/server/src/binary/handlers/streams/update_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs
@@ -19,18 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::update_stream::UpdateStream;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -49,60 +44,18 @@ impl ServerCommandHandler for UpdateStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = shard.resolve_stream_id(&self.stream_id)?;
- shard
- .metadata
- .perm_update_stream(session.get_user_id(), stream_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::UpdateStream {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- name: self.name.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::UpdateStream {
- stream_id, name, ..
- } = payload
- {
- shard.update_stream(&stream_id, name.clone())?;
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected an UpdateStream request inside of UpdateStream handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateStreamResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::UpdateStreamResponse => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected an UpdateStreamResponse inside of UpdateStream handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected UpdateStreamResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/system/get_stats_handler.rs b/core/server/src/binary/handlers/system/get_stats_handler.rs
index 40e6b2024..6ec310920 100644
--- a/core/server/src/binary/handlers/system/get_stats_handler.rs
+++ b/core/server/src/binary/handlers/system/get_stats_handler.rs
@@ -19,19 +19,15 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::system::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
+use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_stats::GetStats;
-use iggy_common::{Identifier, IggyError};
use std::rc::Rc;
use tracing::debug;
@@ -51,37 +47,16 @@ impl ServerCommandHandler for GetStats {
shard.ensure_authenticated(session)?;
shard.metadata.perm_get_stats(session.get_user_id())?;
- // Route GetStats to shard0 only
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::GetStats {
- user_id: session.get_user_id(),
- },
- };
+ let request = ShardRequest::control_plane(ShardRequestPayload::GetStats {
+ user_id: session.get_user_id(),
+ });
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(_) => {
- let stats = shard.get_stats().await.error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to get stats, session: {session}")
- })?;
- let bytes = mapper::map_stats(&stats);
- sender.send_ok_response(&bytes).await?;
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::GetStatsResponse(stats) => {
+ sender.send_ok_response(&mapper::map_stats(&stats)).await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::GetStatsResponse(stats) => {
- let bytes = mapper::map_stats(&stats);
- sender.send_ok_response(&bytes).await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a GetStatsResponse inside of GetStats handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected GetStatsResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index 14f9a3ad8..dbadbb07f 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -19,21 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateTopicWithId;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::create_topic::CreateTopic;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -44,7 +37,7 @@ impl ServerCommandHandler for CreateTopic {
#[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string()))]
async fn handle(
- mut self,
+ self,
sender: &mut SenderKind,
_length: u32,
session: &Session,
@@ -52,112 +45,20 @@ impl ServerCommandHandler for CreateTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = shard.resolve_stream_id(&self.stream_id)?;
- shard
- .metadata
- .perm_create_topic(session.get_user_id(), stream_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::CreateTopic {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- name: self.name.clone(),
- partitions_count: self.partitions_count,
- message_expiry: self.message_expiry,
- compression_algorithm: self.compression_algorithm,
- max_topic_size: self.max_topic_size,
- replication_factor: self.replication_factor,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreateTopic {
- stream_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- partitions_count,
- ..
- } = payload
- {
- // Acquire topic lock to serialize filesystem operations
- let _topic_guard = shard.fs_locks.topic_lock.lock().await;
-
- let topic_id = shard
- .create_topic(
- &stream_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )
- .await?;
-
- let stream_id_num = shard.resolve_stream_id(&stream_id)?;
-
- if let Some((expiry, max_size)) =
- shard.metadata.get_topic_config(stream_id_num, topic_id)
- {
- self.message_expiry = expiry;
- self.max_topic_size = max_size;
- }
- let partition_infos = shard
- .create_partitions(
- &stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- partitions_count,
- )
- .await?;
- let event = ShardEvent::CreatedPartitions {
- stream_id: stream_id.clone(),
- topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions: partition_infos.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- let response = shard.get_topic_from_metadata(stream_id_num, topic_id);
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId {
- topic_id: topic_id as u32,
- command: self
- }))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id_num}, topic_id: {topic_id:?}"
- )
- })?;
- sender.send_ok_response(&response).await?;
- } else {
- unreachable!(
- "Expected a CreateTopic request inside of CreateTopic handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateTopicResponse(data) => {
+ sender
+ .send_ok_response(&mapper::map_topic_from_response(&data))
+ .await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateTopicResponse(topic_id) => {
- let stream_id_num = shard.resolve_stream_id(&self.stream_id)?;
- let response = shard.get_topic_from_metadata(stream_id_num, topic_id);
- sender.send_ok_response(&response).await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a CreateTopicResponse inside of CreateTopic handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreateTopicResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index b81f1ad9d..6305be292 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -19,23 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_topic::DeleteTopic;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
-use tracing::info;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteTopic {
@@ -53,86 +44,18 @@ impl ServerCommandHandler for DeleteTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_delete_topic(session.get_user_id(), stream_id, topic_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::DeleteTopic {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::DeleteTopic {
- stream_id,
- topic_id,
- ..
- } = payload
- {
- // Capture numeric IDs and partition_ids BEFORE deletion for broadcast.
- // After deletion, the topic won't exist in metadata.
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
- let partition_ids = shard
- .metadata
- .get_partition_ids(numeric_stream_id, numeric_topic_id);
-
- // Acquire topic lock to serialize filesystem operations
- let _topic_guard = shard.fs_locks.topic_lock.lock().await;
-
- let topic_info = shard.delete_topic(&stream_id, &topic_id).await?;
- let topic_id_num = topic_info.id;
- let stream_id_num = topic_info.stream_id;
- info!(
- "Deleted topic with name: {}, ID: {} in stream with ID: {}",
- topic_info.name, topic_id_num, stream_id_num
- );
- // Broadcast to all shards to clean up their local_partitions entries.
- // Use numeric Identifiers since the topic is already deleted from metadata.
- let event = ShardEvent::DeletedPartitions {
- stream_id: Identifier::numeric(numeric_stream_id as u32).unwrap(),
- topic_id: Identifier::numeric(numeric_topic_id as u32).unwrap(),
- partitions_count: partition_ids.len() as u32,
- partition_ids,
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
- .await
- .error(|e: &IggyError| format!(
- "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {topic_id_num} in stream with ID: {stream_id_num}, session: {session}",
- ))?;
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a DeleteTopic request inside of DeleteTopic handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteTopicResponse(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteTopicResponse(_topic_id_num) => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a DeleteTopicResponse inside of DeleteTopic handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeleteTopicResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs
index f79a30e37..052cebb2c 100644
--- a/core/server/src/binary/handlers/topics/get_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs
@@ -20,6 +20,7 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
use iggy_common::IggyError;
@@ -43,29 +44,16 @@ impl ServerCommandHandler for GetTopic {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- };
-
- let Some(numeric_topic_id) = shard
- .metadata
- .get_topic_id(numeric_stream_id, &self.topic_id)
+ let Some(topic) =
+ shard
+ .metadata
+ .query_topic(session.get_user_id(), &self.stream_id, &self.topic_id)?
else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
};
- let has_permission = shard
- .metadata
- .perm_get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id)
- .is_ok();
- if !has_permission {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- }
-
- let response = shard.get_topic_from_metadata(numeric_stream_id, numeric_topic_id);
+ let response = mapper::map_topic(&topic);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs
index 9e569e06a..41987f3f4 100644
--- a/core/server/src/binary/handlers/topics/get_topics_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs
@@ -20,6 +20,7 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
+use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
use iggy_common::IggyError;
@@ -43,16 +44,15 @@ impl ServerCommandHandler for GetTopics {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
+ let Some(topics) = shard
+ .metadata
+ .query_topics(session.get_user_id(), &self.stream_id)?
+ else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
};
- shard
- .metadata
- .perm_get_topics(session.get_user_id(), numeric_stream_id)?;
-
- let response = shard.get_topics_from_metadata(numeric_stream_id);
+ let response = mapper::map_topics(&topics);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
index e497ff4d9..1c1281559 100644
--- a/core/server/src/binary/handlers/topics/purge_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
@@ -19,19 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::purge_topic::PurgeTopic;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -50,77 +44,18 @@ impl ServerCommandHandler for PurgeTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_purge_topic(session.get_user_id(), stream_id, topic_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::PurgeTopic {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- },
- };
-
- let stream_id_for_log = self.stream_id.clone();
- let topic_id_for_log = self.topic_id.clone();
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::PurgeTopic {
- stream_id,
- topic_id,
- ..
- } = payload
- {
- shard
- .purge_topic(&stream_id, &topic_id)
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to purge topic with id: {topic_id_for_log}, stream_id: {stream_id_for_log}"
- )
- })?;
-
- let event = ShardEvent::PurgedTopic {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::PurgeTopic(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply purge topic with id: {topic_id_for_log}, stream_id: {stream_id_for_log}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a PurgeTopic request inside of PurgeTopic handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::PurgeTopicResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::PurgeTopicResponse => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a PurgeTopicResponse inside of PurgeTopic handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected PurgeTopicResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs
index a57380a39..cb481e5c8 100644
--- a/core/server/src/binary/handlers/topics/update_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs
@@ -19,19 +19,13 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::update_topic::UpdateTopic;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -42,7 +36,7 @@ impl ServerCommandHandler for UpdateTopic {
#[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_stream_id = self.stream_id.as_string(), iggy_topic_id = self.topic_id.as_string()))]
async fn handle(
- mut self,
+ self,
sender: &mut SenderKind,
_length: u32,
session: &Session,
@@ -50,90 +44,18 @@ impl ServerCommandHandler for UpdateTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (stream_id, topic_id) = shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- shard
- .metadata
- .perm_update_topic(session.get_user_id(), stream_id, topic_id)?;
-
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::UpdateTopic {
- user_id: session.get_user_id(),
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- name: self.name.clone(),
- message_expiry: self.message_expiry,
- compression_algorithm: self.compression_algorithm,
- max_topic_size: self.max_topic_size,
- replication_factor: self.replication_factor,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::UpdateTopic {
- stream_id,
- topic_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- ..
- } = payload
- {
- // Get numeric IDs BEFORE update (name might change during update)
- let (stream_id_num, topic_id_num) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
-
- shard.update_topic(
- &stream_id,
- &topic_id,
- name.clone(),
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )?;
- let (expiry, max_size) = shard
- .metadata
- .get_topic_config(stream_id_num, topic_id_num)
- .ok_or_else(|| {
- IggyError::TopicIdNotFound(topic_id.clone(), stream_id.clone())
- })?;
- self.message_expiry = expiry;
- self.max_topic_size = max_size;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest {
+ user_id: session.get_user_id(),
+ command: self,
+ });
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdateTopic(self))
- .await
- .error(|e: &IggyError| format!(
- "{COMPONENT} (error: {e}) - failed to apply update topic with id: {topic_id} in stream with ID: {stream_id_num}, session: {session}"
- ))?;
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected an UpdateTopic request inside of UpdateTopic handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateTopicResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::UpdateTopicResponse => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected an UpdateTopicResponse inside of UpdateTopic handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected UpdateTopicResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs
index a7657cbf0..9639d68ae 100644
--- a/core/server/src/binary/handlers/users/change_password_handler.rs
+++ b/core/server/src/binary/handlers/users/change_password_handler.rs
@@ -19,21 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use crate::streaming::utils::crypto;
-use err_trail::ErrContext;
use iggy_common::change_password::ChangePassword;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
-use tracing::info;
use tracing::{debug, instrument};
impl ServerCommandHandler for ChangePassword {
@@ -52,88 +45,24 @@ impl ServerCommandHandler for ChangePassword {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- // Check if user is changing someone else's password
let target_user = shard.get_user(&self.user_id)?;
if target_user.id != session.get_user_id() {
shard.metadata.perm_change_password(session.get_user_id())?;
}
- let user_id_for_log = self.user_id.clone();
- let new_password_hash = crypto::hash_password(&self.new_password);
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::ChangePassword {
- session_user_id: session.get_user_id(),
- user_id: self.user_id.clone(),
- current_password: self.current_password.clone(),
- new_password: self.new_password.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::ChangePassword {
- user_id,
- current_password,
- new_password,
- ..
- } = payload
- {
- info!("Changing password for user with ID: {}...", user_id);
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- shard
- .change_password(&user_id, ¤t_password, &new_password)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to change password for user_id: {}, session: {session}",
- user_id
- )
- })?;
-
- info!("Changed password for user with ID: {}.", user_id_for_log);
-
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::ChangePassword(ChangePassword {
- user_id: user_id_for_log.clone(),
- current_password: "".into(),
- new_password: new_password_hash.clone(),
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}",
- user_id_for_log
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::ChangePassword {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ current_password: self.current_password.clone(),
+ new_password: self.new_password.clone(),
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected a ChangePassword request inside of ChangePassword handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::ChangePasswordResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::ChangePasswordResponse => {
- info!("Changed password for user with ID: {}.", user_id_for_log);
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a ChangePasswordResponse inside of ChangePassword handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected ChangePasswordResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs
index abd973029..110eaf00e 100644
--- a/core/server/src/binary/handlers/users/create_user_handler.rs
+++ b/core/server/src/binary/handlers/users/create_user_handler.rs
@@ -19,21 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateUserWithId;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use crate::streaming::utils::crypto;
-use err_trail::ErrContext;
use iggy_common::create_user::CreateUser;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
use tracing::debug;
use tracing::instrument;
@@ -55,85 +48,20 @@ impl ServerCommandHandler for CreateUser {
shard.ensure_authenticated(session)?;
shard.metadata.perm_create_user(session.get_user_id())?;
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::CreateUser {
- user_id: session.get_user_id(),
- username: self.username.clone(),
- password: self.password.clone(),
- status: self.status,
- permissions: self.permissions.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::CreateUser {
- username,
- password,
- status,
- permissions,
- ..
- } = payload
- {
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard
- .create_user(&username, &password, status, permissions.clone())
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create user with name: {}, session: {}",
- username, session
- )
- })?;
-
- let user_id = user.id;
- let response = mapper::map_user(&user);
-
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreateUser(CreateUserWithId {
- user_id,
- command: CreateUser {
- username: self.username.to_owned(),
- password: crypto::hash_password(&self.password),
- status: self.status,
- permissions: self.permissions.clone(),
- },
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create user with name: {}, session: {session}",
- self.username
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateUser {
+ user_id: session.get_user_id(),
+ username: self.username.clone(),
+ password: self.password.clone(),
+ status: self.status,
+ permissions: self.permissions.clone(),
+ });
- sender.send_ok_response(&response).await?;
- } else {
- unreachable!(
- "Expected a CreateUser request inside of CreateUser handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateUserResponse(user) => {
+ sender.send_ok_response(&mapper::map_user(&user)).await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateUserResponse(user) => {
- let response = mapper::map_user(&user);
- sender.send_ok_response(&response).await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a CreateUserResponse inside of CreateUser handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected CreateUserResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs
index 375aa9f06..eb4831728 100644
--- a/core/server/src/binary/handlers/users/delete_user_handler.rs
+++ b/core/server/src/binary/handlers/users/delete_user_handler.rs
@@ -19,20 +19,14 @@
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::delete_user::DeleteUser;
-use iggy_common::{Identifier, IggyError, SenderKind};
+use iggy_common::{IggyError, SenderKind};
use std::rc::Rc;
-use tracing::info;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteUser {
@@ -52,61 +46,17 @@ impl ServerCommandHandler for DeleteUser {
shard.ensure_authenticated(session)?;
shard.metadata.perm_delete_user(session.get_user_id())?;
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::DeleteUser {
- session_user_id: session.get_user_id(),
- user_id: self.user_id,
- },
- };
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(shard_message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = shard_message
- && let ShardRequestPayload::DeleteUser { user_id, .. } = payload
- {
- info!("Deleting user with ID: {}...", user_id);
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard
- .delete_user(&user_id)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete user with ID: {}, session: {session}",
- user_id
- )
- })?;
-
- info!("Deleted user: {} with ID: {}.", user.username, user.id);
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUser {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id,
+ });
- shard
- .state
- .apply(user.id, &EntryCommand::DeleteUser(DeleteUser { user_id: user.id.try_into().unwrap() }))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete user with ID: {user_id}, session: {session}",
- user_id = user.id,
- session = session
- )
- })?;
- sender.send_empty_ok_response().await?;
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletedUser(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeletedUser(_user) => {
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => {
- unreachable!(
- "Expected a DeleteUser request inside of DeleteUser handler, impossible state"
- );
- }
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected DeletedUser"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/users/get_user_handler.rs b/core/server/src/binary/handlers/users/get_user_handler.rs
index c9d9c62d3..5a40a1171 100644
--- a/core/server/src/binary/handlers/users/get_user_handler.rs
+++ b/core/server/src/binary/handlers/users/get_user_handler.rs
@@ -44,21 +44,16 @@ impl ServerCommandHandler for GetUser {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let Ok(user) = shard.find_user(&self.user_id) else {
- sender.send_empty_ok_response().await?;
- return Ok(HandlerResult::Finished);
- };
- let Some(user) = user else {
+
+ let Some(user) = shard
+ .metadata
+ .query_user(session.get_user_id(), &self.user_id)?
+ else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
};
- // Permission check: only required if user is looking for someone else
- if user.id != session.get_user_id() {
- shard.metadata.perm_get_user(session.get_user_id())?;
- }
-
- let bytes = mapper::map_user(&user);
+ let bytes = mapper::map_user_meta(&user);
sender.send_ok_response(&bytes).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/users/get_users_handler.rs b/core/server/src/binary/handlers/users/get_users_handler.rs
index 32ed6039d..df0d4bbff 100644
--- a/core/server/src/binary/handlers/users/get_users_handler.rs
+++ b/core/server/src/binary/handlers/users/get_users_handler.rs
@@ -44,10 +44,10 @@ impl ServerCommandHandler for GetUsers {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard.metadata.perm_get_users(session.get_user_id())?;
- let users = shard.get_users();
- let users = mapper::map_users(users);
- sender.send_ok_response(&users).await?;
+
+ let users = shard.metadata.query_users(session.get_user_id())?;
+ let response = mapper::map_users_meta(&users);
+ sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs
index 92093f158..1b51a93dc 100644
--- a/core/server/src/binary/handlers/users/update_permissions_handler.rs
+++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs
@@ -16,24 +16,17 @@
* under the License.
*/
-use std::rc::Rc;
-
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::update_permissions::UpdatePermissions;
-use iggy_common::{Identifier, IggyError, SenderKind};
-use tracing::info;
+use iggy_common::{IggyError, SenderKind};
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for UpdatePermissions {
@@ -55,76 +48,23 @@ impl ServerCommandHandler for UpdatePermissions {
.metadata
.perm_update_permissions(session.get_user_id())?;
- // Check if target user is root - cannot change root user permissions
let target_user = shard.get_user(&self.user_id)?;
if target_user.is_root() {
return Err(IggyError::CannotChangePermissions(target_user.id));
}
- let user_id_for_log = self.user_id.clone();
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::UpdatePermissions {
- session_user_id: session.get_user_id(),
- user_id: self.user_id.clone(),
- permissions: self.permissions.clone(),
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::UpdatePermissions {
- user_id,
- permissions,
- ..
- } = payload
- {
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- shard
- .update_permissions(&user_id, permissions)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to update permissions for user_id: {}, session: {session}",
- user_id
- )
- })?;
-
- info!("Updated permissions for user with ID: {}.", user_id_for_log);
-
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdatePermissions(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update permissions for user_id: {}, session: {session}",
- user_id_for_log
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissions {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ permissions: self.permissions.clone(),
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected an UpdatePermissions request inside of UpdatePermissions handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdatePermissionsResponse => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::UpdatePermissionsResponse => {
- info!("Updated permissions for user with ID: {}.", user_id_for_log);
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected an UpdatePermissionsResponse inside of UpdatePermissions handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected UpdatePermissionsResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs
index 2e457e8a8..6f043b27e 100644
--- a/core/server/src/binary/handlers/users/update_user_handler.rs
+++ b/core/server/src/binary/handlers/users/update_user_handler.rs
@@ -16,25 +16,17 @@
* under the License.
*/
-use std::rc::Rc;
-
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
-use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
-use err_trail::ErrContext;
use iggy_common::update_user::UpdateUser;
-use iggy_common::{Identifier, IggyError, SenderKind};
-use tracing::info;
+use iggy_common::{IggyError, SenderKind};
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for UpdateUser {
@@ -54,79 +46,19 @@ impl ServerCommandHandler for UpdateUser {
shard.ensure_authenticated(session)?;
shard.metadata.perm_update_user(session.get_user_id())?;
- let user_id_for_log = self.user_id.clone();
- let request = ShardRequest {
- stream_id: Identifier::default(),
- topic_id: Identifier::default(),
- partition_id: 0,
- payload: ShardRequestPayload::UpdateUser {
- session_user_id: session.get_user_id(),
- user_id: self.user_id.clone(),
- username: self.username.clone(),
- status: self.status,
- },
- };
-
- let message = ShardMessage::Request(request);
- match shard.send_request_to_shard_or_recoil(None, message).await? {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest { payload, .. }) = message
- && let ShardRequestPayload::UpdateUser {
- user_id,
- username,
- status,
- ..
- } = payload
- {
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard
- .update_user(&user_id, username.clone(), status)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to update user with user_id: {}, session: {session}",
- user_id
- )
- })?;
-
- info!("Updated user: {} with ID: {}.", user.username, user.id);
-
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::UpdateUser(UpdateUser {
- user_id: user_id_for_log.clone(),
- username,
- status,
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update user with user_id: {}, session: {session}",
- user_id_for_log
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUser {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ username: self.username.clone(),
+ status: self.status,
+ });
- sender.send_empty_ok_response().await?;
- } else {
- unreachable!(
- "Expected an UpdateUser request inside of UpdateUser handler, impossible state"
- );
- }
+ match shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateUserResponse(_) => {
+ sender.send_empty_ok_response().await?;
}
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::UpdateUserResponse(user) => {
- info!("Updated user: {} with ID: {}.", user.username, user.id);
- sender.send_empty_ok_response().await?;
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected an UpdateUserResponse inside of UpdateUser handler, impossible state"
- ),
- },
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected UpdateUserResponse"),
}
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 89dac029b..e6590901f 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -16,7 +16,10 @@
* under the License.
*/
-use crate::metadata::ConsumerGroupMeta;
+use crate::metadata::{ConsumerGroupMeta, StreamMeta, TopicMeta, UserMeta};
+use crate::shard::transmission::frame::{
+ ConsumerGroupResponseData, StreamResponseData, TopicResponseData,
+};
use crate::streaming::clients::client_manager::Client;
use crate::streaming::users::user::User;
use bytes::{BufMut, Bytes, BytesMut};
@@ -143,6 +146,44 @@ pub fn map_personal_access_tokens(personal_access_tokens: Vec<PersonalAccessToke
bytes.freeze()
}
+pub fn map_stream_from_response(data: &StreamResponseData) -> Bytes {
+ let mut bytes = BytesMut::new();
+ bytes.put_u32_le(data.id);
+ bytes.put_u64_le(data.created_at.into());
+ bytes.put_u32_le(0); // topics_count = 0 for new stream
+ bytes.put_u64_le(0); // total_size = 0
+ bytes.put_u64_le(0); // total_messages = 0
+ bytes.put_u8(data.name.len() as u8);
+ bytes.put_slice(data.name.as_bytes());
+ bytes.freeze()
+}
+
+pub fn map_topic_from_response(data: &TopicResponseData) -> Bytes {
+ let mut bytes = BytesMut::new();
+ bytes.put_u32_le(data.id);
+ bytes.put_u64_le(data.created_at.into());
+ bytes.put_u32_le(data.partitions_count);
+ bytes.put_u64_le(data.message_expiry.into());
+ bytes.put_u8(data.compression_algorithm.as_code());
+ bytes.put_u64_le(data.max_topic_size.into());
+ bytes.put_u8(data.replication_factor);
+ bytes.put_u64_le(0); // topic_size = 0
+ bytes.put_u64_le(0); // topic_messages = 0
+ bytes.put_u8(data.name.len() as u8);
+ bytes.put_slice(data.name.as_bytes());
+ bytes.freeze()
+}
+
+pub fn map_consumer_group_from_response(data: &ConsumerGroupResponseData) -> Bytes {
+ let mut bytes = BytesMut::new();
+ bytes.put_u32_le(data.id);
+ bytes.put_u32_le(data.partitions_count);
+ bytes.put_u32_le(0); // members_count = 0 for new group
+ bytes.put_u8(data.name.len() as u8);
+ bytes.put_slice(data.name.as_bytes());
+ bytes.freeze()
+}
+
/// Map consumer group from SharedMetadata format.
pub fn map_consumer_group_from_meta(meta: &ConsumerGroupMeta) -> Bytes {
let mut bytes = BytesMut::new();
@@ -202,3 +243,175 @@ fn extend_pat(personal_access_token: &PersonalAccessToken, bytes: &mut BytesMut)
}
}
}
+
+fn compute_stream_stats(stream: &StreamMeta) -> (u64, u64) {
+ let mut size = 0u64;
+ let mut messages = 0u64;
+ for (_, topic) in stream.topics.iter() {
+ for partition in topic.partitions.iter() {
+ size += partition.stats.size_bytes_inconsistent();
+ messages += partition.stats.messages_count_inconsistent();
+ }
+ }
+ (size, messages)
+}
+
+fn compute_topic_stats(topic: &TopicMeta) -> (u64, u64) {
+ let mut size = 0u64;
+ let mut messages = 0u64;
+ for partition in topic.partitions.iter() {
+ size += partition.stats.size_bytes_inconsistent();
+ messages += partition.stats.messages_count_inconsistent();
+ }
+ (size, messages)
+}
+
+fn extend_topic_header(topic: &TopicMeta, bytes: &mut BytesMut) {
+ let (size, messages) = compute_topic_stats(topic);
+ bytes.put_u32_le(topic.id as u32);
+ bytes.put_u64_le(topic.created_at.into());
+ bytes.put_u32_le(topic.partitions.len() as u32);
+ bytes.put_u64_le(topic.message_expiry.into());
+ bytes.put_u8(topic.compression_algorithm.as_code());
+ bytes.put_u64_le(topic.max_topic_size.into());
+ bytes.put_u8(topic.replication_factor);
+ bytes.put_u64_le(size);
+ bytes.put_u64_le(messages);
+ bytes.put_u8(topic.name.len() as u8);
+ bytes.put_slice(topic.name.as_bytes());
+}
+
+fn extend_topic_with_partitions(topic: &TopicMeta, bytes: &mut BytesMut) {
+ extend_topic_header(topic, bytes);
+
+ for (partition_id, partition) in topic.partitions.iter().enumerate() {
+ let created_at = partition.created_at;
+ let segments_count = partition.stats.segments_count_inconsistent();
+ let offset = partition.stats.current_offset();
+ let size_bytes = partition.stats.size_bytes_inconsistent();
+ let messages_count = partition.stats.messages_count_inconsistent();
+
+ bytes.put_u32_le(partition_id as u32);
+ bytes.put_u64_le(created_at.into());
+ bytes.put_u32_le(segments_count);
+ bytes.put_u64_le(offset);
+ bytes.put_u64_le(size_bytes);
+ bytes.put_u64_le(messages_count);
+ }
+}
+
+/// Map a single stream metadata to wire format (includes nested topics).
+pub fn map_stream(stream: &StreamMeta) -> Bytes {
+ let mut bytes = BytesMut::new();
+
+ let mut topic_ids: Vec<_> = stream.topics.iter().map(|(k, _)| k).collect();
+ topic_ids.sort_unstable();
+
+ let (total_size, total_messages) = compute_stream_stats(stream);
+
+ bytes.put_u32_le(stream.id as u32);
+ bytes.put_u64_le(stream.created_at.into());
+ bytes.put_u32_le(topic_ids.len() as u32);
+ bytes.put_u64_le(total_size);
+ bytes.put_u64_le(total_messages);
+ bytes.put_u8(stream.name.len() as u8);
+ bytes.put_slice(stream.name.as_bytes());
+
+ for &topic_id in &topic_ids {
+ if let Some(topic) = stream.topics.get(topic_id) {
+ extend_topic_header(topic, &mut bytes);
+ }
+ }
+
+ bytes.freeze()
+}
+
+/// Map multiple streams to wire format (header only, no nested topics).
+pub fn map_streams(streams: &[StreamMeta]) -> Bytes {
+ let mut bytes = BytesMut::new();
+
+ let mut sorted: Vec<_> = streams.iter().collect();
+ sorted.sort_by_key(|s| s.id);
+
+ for stream in sorted {
+ let (total_size, total_messages) = compute_stream_stats(stream);
+ let topics_count = stream.topics.len();
+
+ bytes.put_u32_le(stream.id as u32);
+ bytes.put_u64_le(stream.created_at.into());
+ bytes.put_u32_le(topics_count as u32);
+ bytes.put_u64_le(total_size);
+ bytes.put_u64_le(total_messages);
+ bytes.put_u8(stream.name.len() as u8);
+ bytes.put_slice(stream.name.as_bytes());
+ }
+
+ bytes.freeze()
+}
+
+/// Map a single topic metadata to wire format (includes partitions).
+pub fn map_topic(topic: &TopicMeta) -> Bytes {
+ let mut bytes = BytesMut::new();
+ extend_topic_with_partitions(topic, &mut bytes);
+ bytes.freeze()
+}
+
+/// Map multiple topics to wire format (header only, no partitions).
+pub fn map_topics(topics: &[TopicMeta]) -> Bytes {
+ let mut bytes = BytesMut::new();
+
+ let mut sorted: Vec<_> = topics.iter().collect();
+ sorted.sort_by_key(|t| t.id);
+
+ for topic in sorted {
+ extend_topic_header(topic, &mut bytes);
+ }
+
+ bytes.freeze()
+}
+
+/// Map multiple consumer groups to wire format.
+pub fn map_consumer_groups(groups: &[ConsumerGroupMeta]) -> Bytes {
+ let mut bytes = BytesMut::new();
+ for cg in groups {
+ bytes.put_u32_le(cg.id as u32);
+ bytes.put_u32_le(cg.partitions.len() as u32);
+ bytes.put_u32_le(cg.members.len() as u32);
+ bytes.put_u8(cg.name.len() as u8);
+ bytes.put_slice(cg.name.as_bytes());
+ }
+ bytes.freeze()
+}
+
+fn extend_user_meta(user: &UserMeta, bytes: &mut BytesMut) {
+ bytes.put_u32_le(user.id);
+ bytes.put_u64_le(user.created_at.into());
+ bytes.put_u8(user.status.as_code());
+ bytes.put_u8(user.username.len() as u8);
+ bytes.put_slice(user.username.as_bytes());
+}
+
+/// Map a single user metadata to wire format.
+pub fn map_user_meta(user: &UserMeta) -> Bytes {
+ let mut bytes = BytesMut::new();
+ extend_user_meta(user, &mut bytes);
+ if let Some(permissions) = &user.permissions {
+ bytes.put_u8(1);
+ let permissions = permissions.to_bytes();
+ #[allow(clippy::cast_possible_truncation)]
+ bytes.put_u32_le(permissions.len() as u32);
+ bytes.put_slice(&permissions);
+ } else {
+ bytes.put_u32_le(0);
+ }
+ bytes.freeze()
+}
+
+/// Map multiple user metadata to wire format.
+pub fn map_users_meta(users: &[UserMeta]) -> Bytes {
+ let mut bytes = BytesMut::new();
+ for user in users {
+ extend_user_meta(user, &mut bytes);
+ }
+ bytes.freeze()
+}
diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs
index c595de22a..5224392be 100644
--- a/core/server/src/http/consumer_groups.rs
+++ b/core/server/src/http/consumer_groups.rs
@@ -16,26 +16,22 @@
* under the License.
*/
-use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::mapper;
use crate::http::shared::AppState;
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateConsumerGroupWithId;
-use crate::streaming::polling_consumer::ConsumerGroupId;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use axum::debug_handler;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::get;
use axum::{Extension, Json, Router};
-use err_trail::ErrContext;
use iggy_common::Identifier;
use iggy_common::Validatable;
use iggy_common::create_consumer_group::CreateConsumerGroup;
use iggy_common::delete_consumer_group::DeleteConsumerGroup;
-use iggy_common::{ConsumerGroup, ConsumerGroupDetails, IggyError};
-use send_wrapper::SendWrapper;
+use iggy_common::{ConsumerGroup, ConsumerGroupDetails};
use std::sync::Arc;
use tracing::instrument;
@@ -62,21 +58,19 @@ async fn get_consumer_group(
let identifier_group_id = Identifier::from_str_value(&group_id)?;
let shard = state.shard.shard();
- let (numeric_stream_id, numeric_topic_id, numeric_group_id) = shard.resolve_consumer_group_id(
+ let group = shard.resolve_consumer_group(
&identifier_stream_id,
&identifier_topic_id,
&identifier_group_id,
)?;
- shard.metadata.perm_get_consumer_group(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
+ shard
+ .metadata
+ .perm_get_consumer_group(identity.user_id, group.stream_id, group.topic_id)?;
let cg_meta = shard
.metadata
- .get_consumer_group(numeric_stream_id, numeric_topic_id, numeric_group_id)
+ .get_consumer_group(group.stream_id, group.topic_id, group.group_id)
.ok_or(CustomError::ResourceNotFound)?;
let consumer_group = mapper::map_consumer_group_details_from_metadata(&cg_meta);
@@ -93,18 +87,15 @@ async fn get_consumer_groups(
let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
let shard = state.shard.shard();
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
+ let topic = shard.resolve_topic(&identifier_stream_id, &identifier_topic_id)?;
- shard.metadata.perm_get_consumer_groups(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
+ shard
+ .metadata
+ .perm_get_consumer_groups(identity.user_id, topic.stream_id, topic.topic_id)?;
let topic_meta = shard
.metadata
- .get_topic(numeric_stream_id, numeric_topic_id)
+ .get_topic(topic.stream_id, topic.topic_id)
.ok_or(CustomError::ResourceNotFound)?;
let consumer_groups = mapper::map_consumer_groups_from_metadata(&topic_meta);
@@ -123,47 +114,28 @@ async fn create_consumer_group(
command.topic_id = Identifier::from_str_value(&topic_id)?;
command.validate()?;
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&command.stream_id, &command.topic_id)?;
- state.shard.shard().metadata.perm_create_consumer_group(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
-
- let group_id = state.shard.shard().create_consumer_group(
- &command.stream_id,
- &command.topic_id,
- command.name.clone(),
- )
- .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to create consumer group, stream ID: {}, topic ID: {}, name: {}", stream_id, topic_id, command.name))?;
-
let shard = state.shard.shard();
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&command.stream_id, &command.topic_id)?;
- let cg_meta = shard
- .metadata
- .get_consumer_group(numeric_stream_id, numeric_topic_id, group_id)
- .expect("Consumer group must exist after creation");
- let consumer_group_details = mapper::map_consumer_group_details_from_metadata(&cg_meta);
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
- let entry_command = EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
- group_id: group_id as u32,
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest {
+ user_id: identity.user_id,
command,
});
- let state_future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
-
- state_future.await?;
- Ok((StatusCode::CREATED, Json(consumer_group_details)))
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateConsumerGroupResponse(data) => {
+ let cg_meta = state
+ .shard
+ .shard()
+ .metadata
+ .get_consumer_group(topic.stream_id, topic.topic_id, data.id as usize)
+ .expect("Consumer group must exist after creation");
+ let consumer_group_details = mapper::map_consumer_group_details_from_metadata(&cg_meta);
+ Ok((StatusCode::CREATED, Json(consumer_group_details)))
+ }
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreateConsumerGroupResponse"),
+ }
}
#[debug_handler]
@@ -173,88 +145,22 @@ async fn delete_consumer_group(
Extension(identity): Extension<Identity>,
Path((stream_id, topic_id, group_id)): Path<(String, String, String)>,
) -> Result<StatusCode, CustomError> {
- let identifier_stream_id = Identifier::from_str_value(&stream_id)?;
- let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
- let identifier_group_id = Identifier::from_str_value(&group_id)?;
-
- let result = SendWrapper::new(async move {
- let (stream_id_numeric, topic_id_numeric) = state
- .shard
- .shard()
- .resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
- state.shard.shard().metadata.perm_delete_consumer_group(
- identity.user_id,
- stream_id_numeric,
- topic_id_numeric,
- )?;
-
- // Now check if consumer group exists
- state.shard.shard().ensure_consumer_group_exists(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id,
- )?;
-
- let shard = state.shard.shard();
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
-
- let cg_meta = shard.delete_consumer_group(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id
- )
- .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {group_id} for topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
-
- let cg_id = cg_meta.id;
-
- for (_, member) in cg_meta.members.iter() {
- if let Err(err) = shard.client_manager.leave_consumer_group(
- member.client_id,
- numeric_stream_id,
- numeric_topic_id,
- cg_id,
- ) {
- tracing::warn!(
- "{COMPONENT} (error: {err}) - failed to make client leave consumer group for client ID: {}, group ID: {}",
- member.client_id,
- cg_id
- );
- }
- }
-
- let cg_id_spez = ConsumerGroupId(cg_id);
- state.shard.shard().delete_consumer_group_offsets(
- cg_id_spez,
- &identifier_stream_id,
- &identifier_topic_id,
- &cg_meta.partitions,
- ).await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
- cg_id_spez,
- identifier_stream_id,
- identifier_topic_id
- )
- })?;
-
- let entry_command = EntryCommand::DeleteConsumerGroup(DeleteConsumerGroup {
- stream_id: identifier_stream_id,
- topic_id: identifier_topic_id,
- group_id: identifier_group_id,
- });
- let state_future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
-
- state_future.await?;
-
- Ok::<StatusCode, CustomError>(StatusCode::NO_CONTENT)
+ let stream_id = Identifier::from_str_value(&stream_id)?;
+ let topic_id = Identifier::from_str_value(&topic_id)?;
+ let group_id = Identifier::from_str_value(&group_id)?;
+
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest {
+ user_id: identity.user_id,
+ command: DeleteConsumerGroup {
+ stream_id,
+ topic_id,
+ group_id,
+ },
});
- result.await
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteConsumerGroupResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeleteConsumerGroupResponse"),
+ }
}
diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs
index ab7807d48..fb546edd5 100644
--- a/core/server/src/http/consumer_offsets.rs
+++ b/core/server/src/http/consumer_offsets.rs
@@ -61,15 +61,11 @@ async fn get_consumer_offset(
query.topic_id = Identifier::from_str_value(&topic_id)?;
query.validate()?;
- let (stream_id_numeric, topic_id_numeric) = state
- .shard
- .shard()
- .resolve_topic_id(&query.stream_id, &query.topic_id)?;
- state.shard.shard().metadata.perm_get_consumer_offset(
- identity.user_id,
- stream_id_numeric,
- topic_id_numeric,
- )?;
+ let shard = state.shard.shard();
+ let topic = shard.resolve_topic(&query.stream_id, &query.topic_id)?;
+ shard
+ .metadata
+ .perm_get_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?;
let consumer = Consumer::new(query.0.consumer.id);
let Ok(offset) = state
@@ -104,15 +100,11 @@ async fn store_consumer_offset(
command.topic_id = Identifier::from_str_value(&topic_id)?;
command.validate()?;
- let (stream_id_numeric, topic_id_numeric) = state
- .shard
- .shard()
- .resolve_topic_id(&command.stream_id, &command.topic_id)?;
- state.shard.shard().metadata.perm_store_consumer_offset(
- identity.user_id,
- stream_id_numeric,
- topic_id_numeric,
- )?;
+ let shard = state.shard.shard();
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_store_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?;
let consumer = Consumer::new(command.0.consumer.id);
state.shard
@@ -139,14 +131,12 @@ async fn delete_consumer_offset(
let stream_id_ident = Identifier::from_str_value(&stream_id)?;
let topic_id_ident = Identifier::from_str_value(&topic_id)?;
- let (stream_id_numeric, topic_id_numeric) = state
- .shard
- .shard()
- .resolve_topic_id(&stream_id_ident, &topic_id_ident)?;
- state.shard.shard().metadata.perm_delete_consumer_offset(
+ let shard = state.shard.shard();
+ let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?;
+ shard.metadata.perm_delete_consumer_offset(
identity.user_id,
- stream_id_numeric,
- topic_id_numeric,
+ topic.stream_id,
+ topic.topic_id,
)?;
let consumer = Consumer::new(consumer_id.try_into()?);
diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs
index b1f5bc754..f710896d8 100644
--- a/core/server/src/http/http_shard_wrapper.rs
+++ b/core/server/src/http/http_shard_wrapper.rs
@@ -18,30 +18,31 @@
use std::rc::Rc;
use iggy_common::{
- Consumer, ConsumerOffsetInfo, Identifier, IggyError, IggyExpiry, Partitioning,
- PartitioningKind, Permissions, Stats, UserId, UserStatus,
+ Consumer, ConsumerOffsetInfo, Identifier, IggyError, Partitioning, PartitioningKind,
};
use send_wrapper::SendWrapper;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::system::messages::PollingArgs;
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::ShardRequest;
use crate::streaming::segments::{IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::topics;
use crate::streaming::users::user::User;
use crate::{shard::IggyShard, streaming::session::Session};
-use iggy_common::PersonalAccessToken;
-/// A wrapper around IggyShard that is safe to use in HTTP handlers.
+/// Wrapper around IggyShard for HTTP handlers.
+///
+/// Provides three categories of access:
+/// 1. Control-plane mutations via `send_to_control_plane()` (routed through message pump)
+/// 2. Read-only metadata access via `shard()` (direct, same-thread safe)
+/// 3. Data-plane operations (poll/append messages, consumer offsets)
///
/// # Safety
-/// This wrapper is only safe to use when:
-/// 1. The HTTP server runs on a single thread (compio's thread-per-core model)
-/// 2. All operations are confined to shard 0's thread
+/// This wrapper is safe because:
+/// 1. HTTP server runs on shard 0's single thread (compio model)
+/// 2. All operations are confined to that thread
/// 3. The underlying IggyShard is never accessed from multiple threads
-///
-/// The safety guarantee is provided by the HTTP server architecture where
-/// all HTTP requests are handled on the same thread that owns the IggyShard.
pub struct HttpSafeShard {
inner: Rc<IggyShard>,
}
@@ -59,10 +60,22 @@ impl HttpSafeShard {
Self { inner: shard }
}
+ /// Direct access to shard for read-only operations and auth.
pub fn shard(&self) -> &IggyShard {
&self.inner
}
+ /// Route control-plane mutations through the message pump.
+ pub async fn send_to_control_plane(
+ &self,
+ request: ShardRequest,
+ ) -> Result<ShardResponse, IggyError> {
+ let future = SendWrapper::new(self.inner.send_to_control_plane(request));
+ future.await
+ }
+
+ // === Data-plane operations (message polling/appending) ===
+
pub async fn get_consumer_offset(
&self,
client_id: u32,
@@ -71,11 +84,11 @@ impl HttpSafeShard {
topic_id: &Identifier,
partition_id: Option<u32>,
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
+ let topic = self.shard().resolve_topic(stream_id, topic_id)?;
let future = SendWrapper::new(self.shard().get_consumer_offset(
client_id,
consumer,
- stream_id,
- topic_id,
+ topic,
partition_id,
));
future.await
@@ -90,11 +103,11 @@ impl HttpSafeShard {
partition_id: Option<u32>,
offset: u64,
) -> Result<(), IggyError> {
+ let topic = self.shard().resolve_topic(stream_id, topic_id)?;
let future = SendWrapper::new(self.shard().store_consumer_offset(
client_id,
consumer,
- stream_id,
- topic_id,
+ topic,
partition_id,
offset,
));
@@ -110,138 +123,17 @@ impl HttpSafeShard {
topic_id: &Identifier,
partition_id: Option<u32>,
) -> Result<(), IggyError> {
+ let topic = self.shard().resolve_topic(stream_id, topic_id)?;
let future = SendWrapper::new(self.shard().delete_consumer_offset(
client_id,
consumer,
- stream_id,
- topic_id,
+ topic,
partition_id,
));
let _result = future.await?;
Ok(())
}
- pub async fn delete_stream(&self, stream_id: &Identifier) -> Result<(), IggyError> {
- let future = SendWrapper::new(self.shard().delete_stream(stream_id));
- future.await?;
- Ok(())
- }
-
- pub fn update_stream(&self, stream_id: &Identifier, name: String) -> Result<(), IggyError> {
- self.shard().update_stream(stream_id, name)
- }
-
- pub async fn purge_stream(&self, stream_id: &Identifier) -> Result<(), IggyError> {
- let future = SendWrapper::new(self.shard().purge_stream(stream_id));
- future.await
- }
-
- pub async fn create_stream(&self, name: String) -> Result<usize, IggyError> {
- let future = SendWrapper::new(self.shard().create_stream(name));
- future.await
- }
-
- pub async fn apply_state(
- &self,
- user_id: UserId,
- command: &EntryCommand,
- ) -> Result<(), IggyError> {
- self.shard().state.apply(user_id, command).await
- }
-
- pub fn get_users(&self) -> Vec<User> {
- self.shard().get_users()
- }
-
- pub fn create_user(
- &self,
- username: &str,
- password: &str,
- status: UserStatus,
- permissions: Option<Permissions>,
- ) -> Result<User, IggyError> {
- self.shard()
- .create_user(username, password, status, permissions)
- }
-
- pub fn delete_user(&self, user_id: &Identifier) -> Result<User, IggyError> {
- self.shard().delete_user(user_id)
- }
-
- pub fn update_user(
- &self,
- user_id: &Identifier,
- username: Option<String>,
- status: Option<UserStatus>,
- ) -> Result<User, IggyError> {
- self.shard().update_user(user_id, username, status)
- }
-
- pub fn update_permissions(
- &self,
- user_id: &Identifier,
- permissions: Option<Permissions>,
- ) -> Result<(), IggyError> {
- self.shard().update_permissions(user_id, permissions)
- }
-
- pub fn change_password(
- &self,
- user_id: &Identifier,
- current_password: &str,
- new_password: &str,
- ) -> Result<(), IggyError> {
- self.shard()
- .change_password(user_id, current_password, new_password)
- }
-
- pub fn login_user(
- &self,
- username: &str,
- password: &str,
- session: Option<&Session>,
- ) -> Result<User, IggyError> {
- self.shard().login_user(username, password, session)
- }
-
- pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
- self.shard().logout_user(session)
- }
-
- pub fn get_personal_access_tokens(
- &self,
- user_id: u32,
- ) -> Result<Vec<PersonalAccessToken>, IggyError> {
- self.shard().get_personal_access_tokens(user_id)
- }
-
- pub fn create_personal_access_token(
- &self,
- user_id: u32,
- name: &str,
- expiry: IggyExpiry,
- ) -> Result<(PersonalAccessToken, String), IggyError> {
- self.shard()
- .create_personal_access_token(user_id, name, expiry)
- }
-
- pub fn delete_personal_access_token(&self, user_id: u32, name: &str) -> Result<(), IggyError> {
- self.shard().delete_personal_access_token(user_id, name)
- }
-
- pub fn login_with_personal_access_token(
- &self,
- token: &str,
- session: Option<&Session>,
- ) -> Result<User, IggyError> {
- self.shard()
- .login_with_personal_access_token(token, session)
- }
-
- pub async fn get_stats(&self) -> Result<Stats, IggyError> {
- self.shard().get_stats().await
- }
-
#[allow(clippy::too_many_arguments)]
pub async fn poll_messages(
&self,
@@ -253,11 +145,12 @@ impl HttpSafeShard {
maybe_partition_id: Option<u32>,
args: PollingArgs,
) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> {
+ let topic = self
+ .shard()
+ .resolve_topic_for_poll(user_id, &stream_id, &topic_id)?;
let future = SendWrapper::new(self.shard().poll_messages(
client_id,
- user_id,
- stream_id,
- topic_id,
+ topic,
consumer.clone(),
maybe_partition_id,
args,
@@ -274,27 +167,17 @@ impl HttpSafeShard {
partitioning: &Partitioning,
batch: IggyMessagesBatchMut,
) -> Result<(), IggyError> {
- self.shard().ensure_topic_exists(&stream_id, &topic_id)?;
+ use crate::shard::transmission::message::ResolvedPartition;
- let numeric_stream_id = self
+ let topic = self
.shard()
- .metadata
- .get_stream_id(&stream_id)
- .expect("Stream existence already verified");
- let numeric_topic_id = self
- .shard()
- .metadata
- .get_topic_id(numeric_stream_id, &topic_id)
- .expect("Topic existence already verified");
+ .resolve_topic_for_append(user_id, &stream_id, &topic_id)?;
let partition_id = match partitioning.kind {
PartitioningKind::Balanced => self
.shard()
.metadata
- .get_next_partition_id(numeric_stream_id, numeric_topic_id)
- .ok_or(IggyError::TopicIdNotFound(
- stream_id.clone(),
- topic_id.clone(),
- ))?,
+ .get_next_partition_id(topic.stream_id, topic.topic_id)
+ .ok_or(IggyError::TopicIdNotFound(stream_id, topic_id))?,
PartitioningKind::PartitionId => u32::from_le_bytes(
partitioning.value[..partitioning.length as usize]
.try_into()
@@ -304,7 +187,7 @@ impl HttpSafeShard {
let partitions_count = self
.shard()
.metadata
- .partitions_count(numeric_stream_id, numeric_topic_id);
+ .partitions_count(topic.stream_id, topic.topic_id);
topics::helpers::calculate_partition_id_by_messages_key_hash(
partitions_count,
&partitioning.value,
@@ -312,45 +195,35 @@ impl HttpSafeShard {
}
};
- let future = SendWrapper::new(self.shard().append_messages(
- user_id,
- stream_id,
- topic_id,
+ let partition = ResolvedPartition {
+ stream_id: topic.stream_id,
+ topic_id: topic.topic_id,
partition_id,
- batch,
- ));
+ };
+
+ let future = SendWrapper::new(self.shard().append_messages(partition, batch));
future.await
}
- pub fn create_consumer_group(
+ pub fn login_user(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- name: String,
- ) -> Result<usize, IggyError> {
- self.shard()
- .create_consumer_group(stream_id, topic_id, name)
+ username: &str,
+ password: &str,
+ session: Option<&Session>,
+ ) -> Result<User, IggyError> {
+ self.shard().login_user(username, password, session)
}
- #[allow(clippy::too_many_arguments)]
- pub fn update_topic(
+ pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
+ self.shard().logout_user(session)
+ }
+
+ pub fn login_with_personal_access_token(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- name: String,
- message_expiry: iggy_common::IggyExpiry,
- compression_algorithm: iggy_common::CompressionAlgorithm,
- max_topic_size: iggy_common::MaxTopicSize,
- replication_factor: Option<u8>,
- ) -> Result<(), IggyError> {
- self.shard().update_topic(
- stream_id,
- topic_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )
+ token: &str,
+ session: Option<&Session>,
+ ) -> Result<User, IggyError> {
+ self.shard()
+ .login_with_personal_access_token(token, session)
}
}
diff --git a/core/server/src/http/messages.rs b/core/server/src/http/messages.rs
index 008175eab..125c0ad25 100644
--- a/core/server/src/http/messages.rs
+++ b/core/server/src/http/messages.rs
@@ -21,6 +21,7 @@ use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
use crate::shard::system::messages::PollingArgs;
+use crate::shard::transmission::message::ResolvedPartition;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
use axum::extract::{Path, Query, State};
@@ -131,17 +132,20 @@ async fn flush_unsaved_buffer(
Extension(identity): Extension<Identity>,
Path((stream_id, topic_id, partition_id, fsync)): Path<(String, String, u32, bool)>,
) -> Result<StatusCode, CustomError> {
- let stream_id = Identifier::from_str_value(&stream_id)?;
- let topic_id = Identifier::from_str_value(&topic_id)?;
+ let stream_id_ident = Identifier::from_str_value(&stream_id)?;
+ let topic_id_ident = Identifier::from_str_value(&topic_id)?;
let partition_id = partition_id as usize;
- let flush_future = SendWrapper::new(state.shard.shard().flush_unsaved_buffer(
- identity.user_id,
- stream_id,
- topic_id,
+ let shard = state.shard.shard();
+ let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?;
+ let partition = ResolvedPartition {
+ stream_id: topic.stream_id,
+ topic_id: topic.topic_id,
partition_id,
- fsync,
- ));
+ };
+
+ let flush_future =
+ SendWrapper::new(shard.flush_unsaved_buffer(identity.user_id, partition, fsync));
flush_future.await?;
Ok(StatusCode::OK)
}
diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs
index 2c5a2abf5..96570a711 100644
--- a/core/server/src/http/partitions.rs
+++ b/core/server/src/http/partitions.rs
@@ -16,23 +16,19 @@
* under the License.
*/
-use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::shard::transmission::event::ShardEvent;
-use crate::state::command::EntryCommand;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
use axum::routing::post;
use axum::{Extension, Json, Router, debug_handler};
-use err_trail::ErrContext;
use iggy_common::Identifier;
-use iggy_common::IggyError;
use iggy_common::Validatable;
use iggy_common::create_partitions::CreatePartitions;
use iggy_common::delete_partitions::DeletePartitions;
-use send_wrapper::SendWrapper;
use std::sync::Arc;
use tracing::instrument;
@@ -57,54 +53,16 @@ async fn create_partitions(
command.topic_id = Identifier::from_str_value(&topic_id)?;
command.validate()?;
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&command.stream_id, &command.topic_id)?;
- state.shard.shard().metadata.perm_create_partitions(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
-
- let _parititon_guard = state.shard.shard().fs_locks.partition_lock.lock().await;
- let partition_infos = SendWrapper::new(state.shard.shard().create_partitions(
- &command.stream_id,
- &command.topic_id,
- command.partitions_count,
- ))
- .await?;
-
- let broadcast_future = SendWrapper::new(async {
- let shard = state.shard.shard();
-
- let event = ShardEvent::CreatedPartitions {
- stream_id: command.stream_id.clone(),
- topic_id: command.topic_id.clone(),
- partitions: partition_infos,
- };
- let _responses = shard.broadcast_event_to_all_shards(event).await;
- Ok::<(), CustomError>(())
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest {
+ user_id: identity.user_id,
+ command,
});
- broadcast_future.await
- .error(|e: &CustomError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to broadcast partition events, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
- let command = EntryCommand::CreatePartitions(command);
- let state_future =
- SendWrapper::new(state.shard.shard().state.apply(identity.user_id, &command));
-
- state_future.await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create partitions, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
-
- Ok(StatusCode::CREATED)
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreatePartitionsResponse(_) => Ok(StatusCode::CREATED),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreatePartitionsResponse"),
+ }
}
#[debug_handler]
@@ -119,62 +77,18 @@ async fn delete_partitions(
query.topic_id = Identifier::from_str_value(&topic_id)?;
query.validate()?;
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&query.stream_id, &query.topic_id)?;
- state.shard.shard().metadata.perm_delete_partitions(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
-
- let deleted_partition_ids = {
- let delete_future = SendWrapper::new(state.shard.shard().delete_partitions(
- &query.stream_id,
- &query.topic_id,
- query.partitions_count,
- ));
-
- delete_future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete partitions for topic with ID: {topic_id} in stream with ID: {stream_id}"
- )
- })?
- };
-
- // Send event for partition deletion
- {
- let broadcast_future = SendWrapper::new(async {
- let event = ShardEvent::DeletedPartitions {
- stream_id: query.stream_id.clone(),
- topic_id: query.topic_id.clone(),
- partitions_count: query.partitions_count,
- partition_ids: deleted_partition_ids,
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
- let command = EntryCommand::DeletePartitions(DeletePartitions {
- stream_id: query.stream_id.clone(),
- topic_id: query.topic_id.clone(),
- partitions_count: query.partitions_count,
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest {
+ user_id: identity.user_id,
+ command: DeletePartitions {
+ stream_id: query.stream_id.clone(),
+ topic_id: query.topic_id.clone(),
+ partitions_count: query.partitions_count,
+ },
});
- let state_future =
- SendWrapper::new(state.shard.shard().state.apply(identity.user_id, &command));
-
- state_future.await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete partitions, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
- Ok(StatusCode::NO_CONTENT)
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletePartitionsResponse(_) => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeletePartitionsResponse"),
+ }
}
diff --git a/core/server/src/http/personal_access_tokens.rs b/core/server/src/http/personal_access_tokens.rs
index 10f05f961..9301b9345 100644
--- a/core/server/src/http/personal_access_tokens.rs
+++ b/core/server/src/http/personal_access_tokens.rs
@@ -22,21 +22,19 @@ use crate::http::jwt::json_web_token::Identity;
use crate::http::mapper;
use crate::http::mapper::map_generated_access_token_to_identity_info;
use crate::http::shared::AppState;
-use crate::state::command::EntryCommand;
-use crate::state::models::CreatePersonalAccessTokenWithHash;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get, post};
use axum::{Extension, Json, Router, debug_handler};
use err_trail::ErrContext;
use iggy_common::IdentityInfo;
-use iggy_common::PersonalAccessToken;
+use iggy_common::PersonalAccessTokenInfo;
use iggy_common::Validatable;
use iggy_common::create_personal_access_token::CreatePersonalAccessToken;
-use iggy_common::delete_personal_access_token::DeletePersonalAccessToken;
use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken;
-use iggy_common::{IggyError, PersonalAccessTokenInfo, RawPersonalAccessToken};
-use send_wrapper::SendWrapper;
+use iggy_common::{IggyError, RawPersonalAccessToken};
use std::sync::Arc;
use tracing::instrument;
@@ -84,32 +82,20 @@ async fn create_personal_access_token(
Json(command): Json<CreatePersonalAccessToken>,
) -> Result<Json<RawPersonalAccessToken>, CustomError> {
command.validate()?;
- let (_, token) = state
- .shard
- .create_personal_access_token(identity.user_id, &command.name, command.expiry)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create personal access token, user ID: {}",
- identity.user_id
- )
- })?;
- let token_hash = PersonalAccessToken::hash_token(&token);
- let command = EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash {
- command,
- hash: token_hash,
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessToken {
+ user_id: identity.user_id,
+ name: command.name,
+ expiry: command.expiry,
});
- let state_future =
- SendWrapper::new(state.shard.shard().state.apply(identity.user_id, &command));
- state_future.await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create personal access token with hash, user ID: {}",
- identity.user_id
- )
- })?;
- Ok(Json(RawPersonalAccessToken { token }))
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreatePersonalAccessTokenResponse(_, token) => {
+ Ok(Json(RawPersonalAccessToken { token }))
+ }
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreatePersonalAccessTokenResponse"),
+ }
}
#[debug_handler]
@@ -119,28 +105,16 @@ async fn delete_personal_access_token(
Extension(identity): Extension<Identity>,
Path(name): Path<String>,
) -> Result<StatusCode, CustomError> {
- state
- .shard
- .delete_personal_access_token(identity.user_id, &name)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete personal access token, user ID: {}",
- identity.user_id
- )
- })?;
-
- let command =
- EntryCommand::DeletePersonalAccessToken(DeletePersonalAccessToken { name: name.clone() });
- let state_future =
- SendWrapper::new(state.shard.shard().state.apply(identity.user_id, &command));
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessToken {
+ user_id: identity.user_id,
+ name,
+ });
- state_future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete personal access token, user ID: {}",
- identity.user_id
- )
- })?;
- Ok(StatusCode::NO_CONTENT)
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletePersonalAccessTokenResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeletePersonalAccessTokenResponse"),
+ }
}
#[instrument(skip_all, name = "trace_login_with_personal_access_token")]
diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs
index 48de93e63..92a623c66 100644
--- a/core/server/src/http/streams.rs
+++ b/core/server/src/http/streams.rs
@@ -16,26 +16,22 @@
* under the License.
*/
-use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get};
use axum::{Extension, Json, Router, debug_handler};
-use err_trail::ErrContext;
use iggy_common::Identifier;
use iggy_common::Validatable;
use iggy_common::create_stream::CreateStream;
use iggy_common::delete_stream::DeleteStream;
use iggy_common::purge_stream::PurgeStream;
use iggy_common::update_stream::UpdateStream;
-use iggy_common::{IggyError, Stream, StreamDetails};
-use send_wrapper::SendWrapper;
-
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateStreamWithId;
+use iggy_common::{Stream, StreamDetails};
use std::sync::Arc;
use tracing::instrument;
@@ -102,53 +98,26 @@ async fn create_stream(
Json(command): Json<CreateStream>,
) -> Result<Json<StreamDetails>, CustomError> {
command.validate()?;
- state
- .shard
- .shard()
- .metadata
- .perm_create_stream(identity.user_id)?;
-
- let result = SendWrapper::new(async move {
- let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
- // Create stream using wrapper method
- let created_stream_id = state
- .shard
- .create_stream(command.name.clone())
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create stream with name: {}",
- command.name
- )
- })?;
-
- let entry_command = EntryCommand::CreateStream(CreateStreamWithId {
- stream_id: created_stream_id as u32,
- command,
- });
-
- state
- .shard
- .apply_state(identity.user_id, &entry_command)
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create stream for id: {:?}",
- created_stream_id
- )
- })?;
-
- let shard = state.shard.shard();
- let stream_meta = shard
- .metadata
- .get_stream(created_stream_id)
- .expect("Stream must exist after creation");
- let response = crate::http::mapper::map_stream_details_from_metadata(&stream_meta);
- Ok::<Json<StreamDetails>, CustomError>(Json(response))
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest {
+ user_id: identity.user_id,
+ command,
});
- result.await
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateStreamResponse(data) => {
+ let stream_meta = state
+ .shard
+ .shard()
+ .metadata
+ .get_stream(data.id as usize)
+ .expect("Stream must exist after creation");
+ let response = crate::http::mapper::map_stream_details_from_metadata(&stream_meta);
+ Ok(Json(response))
+ }
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreateStreamResponse"),
+ }
}
#[debug_handler]
@@ -162,34 +131,16 @@ async fn update_stream(
command.stream_id = Identifier::from_str_value(&stream_id)?;
command.validate()?;
- let stream_id_numeric = state.shard.shard().resolve_stream_id(&command.stream_id)?;
- state
- .shard
- .shard()
- .metadata
- .perm_update_stream(identity.user_id, stream_id_numeric)?;
-
- let result = SendWrapper::new(async move {
- // Update stream using wrapper method
- state
- .shard
- .update_stream(&command.stream_id, command.name.clone())
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to update stream, stream ID: {stream_id}"
- )
- })?;
-
- let entry_command = EntryCommand::UpdateStream(command);
- state.shard.apply_state(identity.user_id, &entry_command).await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update stream, stream ID: {stream_id}"
- )
- })?;
- Ok::<StatusCode, CustomError>(StatusCode::NO_CONTENT)
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest {
+ user_id: identity.user_id,
+ command,
});
- result.await
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateStreamResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected UpdateStreamResponse"),
+ }
}
#[debug_handler]
@@ -199,43 +150,18 @@ async fn delete_stream(
Extension(identity): Extension<Identity>,
Path(stream_id): Path<String>,
) -> Result<StatusCode, CustomError> {
- let identifier_stream_id = Identifier::from_str_value(&stream_id)?;
-
- let stream_id_numeric = state
- .shard
- .shard()
- .resolve_stream_id(&identifier_stream_id)?;
- state
- .shard
- .shard()
- .metadata
- .perm_delete_stream(identity.user_id, stream_id_numeric)?;
-
- let result =
- SendWrapper::new(async move {
- let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
- {
- let future =
- SendWrapper::new(state.shard.shard().delete_stream(&identifier_stream_id));
- future.await
- }
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to delete stream with ID: {stream_id}",)
- })?;
+ let stream_id = Identifier::from_str_value(&stream_id)?;
- let entry_command = EntryCommand::DeleteStream(DeleteStream {
- stream_id: identifier_stream_id,
- });
- state.shard.apply_state(identity.user_id, &entry_command).await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}",
- )
- })?;
- Ok::<StatusCode, CustomError>(StatusCode::NO_CONTENT)
- });
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest {
+ user_id: identity.user_id,
+ command: DeleteStream { stream_id },
+ });
- result.await
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteStreamResponse(_) => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeleteStreamResponse"),
+ }
}
#[debug_handler]
@@ -245,55 +171,16 @@ async fn purge_stream(
Extension(identity): Extension<Identity>,
Path(stream_id): Path<String>,
) -> Result<StatusCode, CustomError> {
- let identifier_stream_id = Identifier::from_str_value(&stream_id)?;
-
- let stream_id_numeric = state
- .shard
- .shard()
- .resolve_stream_id(&identifier_stream_id)?;
- state
- .shard
- .shard()
- .metadata
- .perm_purge_stream(identity.user_id, stream_id_numeric)?;
-
- let result = SendWrapper::new(async move {
- // Purge stream using wrapper method
- state
- .shard
- .purge_stream(&identifier_stream_id)
- .await
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to purge stream, stream ID: {stream_id}")
- })?;
-
- // Send event for stream purge
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::PurgedStream {
- stream_id: identifier_stream_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
+ let stream_id = Identifier::from_str_value(&stream_id)?;
- let entry_command = EntryCommand::PurgeStream(PurgeStream {
- stream_id: identifier_stream_id,
- });
- state.shard.apply_state(identity.user_id, &entry_command).await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply purge stream, stream ID: {stream_id}"
- )
- })?;
- Ok::<StatusCode, CustomError>(StatusCode::NO_CONTENT)
+ let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest {
+ user_id: identity.user_id,
+ command: PurgeStream { stream_id },
});
- result.await
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::PurgeStreamResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected PurgeStreamResponse"),
+ }
}
diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs
index 91a771eba..7e170313b 100644
--- a/core/server/src/http/topics.rs
+++ b/core/server/src/http/topics.rs
@@ -20,8 +20,8 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateTopicWithId;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get};
@@ -34,7 +34,6 @@ use iggy_common::delete_topic::DeleteTopic;
use iggy_common::purge_topic::PurgeTopic;
use iggy_common::update_topic::UpdateTopic;
use iggy_common::{IggyError, Topic, TopicDetails};
-use send_wrapper::SendWrapper;
use std::sync::Arc;
use tracing::instrument;
@@ -140,106 +139,32 @@ async fn create_topic(
command.stream_id = Identifier::from_str_value(&stream_id)?;
command.validate()?;
- let numeric_stream_id = state.shard.shard().resolve_stream_id(&command.stream_id)?;
- state
+ let numeric_stream_id = state
.shard
.shard()
.metadata
- .perm_create_topic(identity.user_id, numeric_stream_id)?;
-
- let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
- let topic_id = {
- let future = SendWrapper::new(state.shard.shard().create_topic(
- &command.stream_id,
- command.name.clone(),
- command.message_expiry,
- command.compression_algorithm,
- command.max_topic_size,
- command.replication_factor,
- ));
- future.await
- }
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to create topic, stream ID: {stream_id}")
- })?;
-
- {
- let shard = state.shard.shard();
- let numeric_stream_id = shard
- .metadata
- .get_stream_id(&command.stream_id)
- .expect("Stream must exist");
- let (expiry, max_size) = shard
- .metadata
- .get_topic_config(numeric_stream_id, topic_id)
- .expect("Topic config must exist after creation");
- command.message_expiry = expiry;
- command.max_topic_size = max_size;
- }
-
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
-
- let shard = state.shard.shard();
-
- // Create partitions
- let partition_infos = shard
- .create_partitions(
- &command.stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- command.partitions_count,
- )
- .await?;
-
- let event = ShardEvent::CreatedPartitions {
- stream_id: command.stream_id.clone(),
- topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions: partition_infos,
- };
- let _responses = shard.broadcast_event_to_all_shards(event).await;
+ .get_stream_id(&command.stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
- Ok::<(), CustomError>(())
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest {
+ user_id: identity.user_id,
+ command,
});
- broadcast_future.await.error(|e: &CustomError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to broadcast topic events, stream ID: {stream_id}"
- )
- })?;
-
- let response = {
- let shard = state.shard.shard();
- let numeric_stream_id = shard
- .metadata
- .get_stream_id(&command.stream_id)
- .expect("Stream must exist");
- let topic_meta = shard
- .metadata
- .get_topic(numeric_stream_id, topic_id)
- .expect("Topic must exist after creation");
- let topic_response = crate::http::mapper::map_topic_details_from_metadata(&topic_meta);
- Json(topic_response)
- };
-
- {
- let entry_command = EntryCommand::CreateTopic(CreateTopicWithId {
- topic_id: topic_id as u32,
- command,
- });
- let future = SendWrapper::new(
- state
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateTopicResponse(data) => {
+ let topic_meta = state
.shard
.shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await
+ .metadata
+ .get_topic(numeric_stream_id, data.id as usize)
+ .expect("Topic must exist after creation");
+ let response = crate::http::mapper::map_topic_details_from_metadata(&topic_meta);
+ Ok(Json(response))
+ }
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreateTopicResponse"),
}
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to apply create topic, stream ID: {stream_id}",)
- })?;
-
- Ok(response)
}
#[debug_handler]
@@ -254,67 +179,16 @@ async fn update_topic(
command.topic_id = Identifier::from_str_value(&topic_id)?;
command.validate()?;
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&command.stream_id, &command.topic_id)?;
- state.shard.shard().metadata.perm_update_topic(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
-
- let name_changed = !command.name.is_empty();
- state.shard.shard().update_topic(
- &command.stream_id,
- &command.topic_id,
- command.name.clone(),
- command.message_expiry,
- command.compression_algorithm,
- command.max_topic_size,
- command.replication_factor,
- ).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to update topic, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
-
- let shard = state.shard.shard();
- let numeric_stream_id = shard
- .metadata
- .get_stream_id(&command.stream_id)
- .expect("Stream must exist");
-
- let topic_id_for_lookup = if name_changed {
- Identifier::named(&command.name.clone()).unwrap()
- } else {
- command.topic_id.clone()
- };
-
- let numeric_topic_id = shard
- .metadata
- .get_topic_id(numeric_stream_id, &topic_id_for_lookup)
- .expect("Topic must exist after update");
-
- let (message_expiry, max_topic_size) = shard
- .metadata
- .get_topic_config(numeric_stream_id, numeric_topic_id)
- .expect("Topic metadata must exist");
- command.message_expiry = message_expiry;
- command.max_topic_size = max_topic_size;
-
- {
- let entry_command = EntryCommand::UpdateTopic(command);
- let future = SendWrapper::new(state.shard.shard().state
- .apply(identity.user_id, &entry_command));
- future.await
- }.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update topic, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest {
+ user_id: identity.user_id,
+ command,
+ });
- Ok(StatusCode::NO_CONTENT)
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateTopicResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected UpdateTopicResponse"),
+ }
}
#[debug_handler]
@@ -324,50 +198,22 @@ async fn delete_topic(
Extension(identity): Extension<Identity>,
Path((stream_id, topic_id)): Path<(String, String)>,
) -> Result<StatusCode, CustomError> {
- let identifier_stream_id = Identifier::from_str_value(&stream_id)?;
- let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
-
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
- state.shard.shard().metadata.perm_delete_topic(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
- let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
-
- {
- let future = SendWrapper::new(state.shard.shard().delete_topic(
- &identifier_stream_id,
- &identifier_topic_id,
- ));
- future.await
- }.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}",
- )
- })?;
-
- {
- let entry_command = EntryCommand::DeleteTopic(DeleteTopic {
- stream_id: identifier_stream_id,
- topic_id: identifier_topic_id,
- });
- let future = SendWrapper::new(state.shard.shard().state
- .apply(
- identity.user_id,
- &entry_command,
- ));
- future.await
- }.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete topic, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
+ let stream_id = Identifier::from_str_value(&stream_id)?;
+ let topic_id = Identifier::from_str_value(&topic_id)?;
+
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest {
+ user_id: identity.user_id,
+ command: DeleteTopic {
+ stream_id,
+ topic_id,
+ },
+ });
- Ok(StatusCode::NO_CONTENT)
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeleteTopicResponse(_) => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeleteTopicResponse"),
+ }
}
#[debug_handler]
@@ -377,63 +223,20 @@ async fn purge_topic(
Extension(identity): Extension<Identity>,
Path((stream_id, topic_id)): Path<(String, String)>,
) -> Result<StatusCode, CustomError> {
- let identifier_stream_id = Identifier::from_str_value(&stream_id)?;
- let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
-
- let (numeric_stream_id, numeric_topic_id) = state
- .shard
- .shard()
- .resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
- state.shard.shard().metadata.perm_purge_topic(
- identity.user_id,
- numeric_stream_id,
- numeric_topic_id,
- )?;
-
- {
- let future = SendWrapper::new(state.shard.shard().purge_topic(
- &identifier_stream_id,
- &identifier_topic_id,
- ));
- future.await
- }.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to purge topic, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
+ let stream_id = Identifier::from_str_value(&stream_id)?;
+ let topic_id = Identifier::from_str_value(&topic_id)?;
+
+ let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest {
+ user_id: identity.user_id,
+ command: PurgeTopic {
+ stream_id,
+ topic_id,
+ },
+ });
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::PurgedTopic {
- stream_id: identifier_stream_id.clone(),
- topic_id: identifier_topic_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::PurgeTopicResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected PurgeTopicResponse"),
}
-
- {
- let entry_command = EntryCommand::PurgeTopic(PurgeTopic {
- stream_id: identifier_stream_id,
- topic_id: identifier_topic_id,
- });
- let future = SendWrapper::new(state.shard.shard().state
- .apply(
- identity.user_id,
- &entry_command,
- ));
- future.await
- }.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply purge topic, stream ID: {stream_id}, topic ID: {topic_id}"
- )
- })?;
-
- Ok(StatusCode::NO_CONTENT)
}
diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs
index 8e4dcd1b9..cb265447f 100644
--- a/core/server/src/http/users.rs
+++ b/core/server/src/http/users.rs
@@ -22,14 +22,12 @@ use crate::http::jwt::json_web_token::Identity;
use crate::http::mapper;
use crate::http::mapper::map_generated_access_token_to_identity_info;
use crate::http::shared::AppState;
-use crate::state::command::EntryCommand;
-use crate::state::models::CreateUserWithId;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::session::Session;
use crate::streaming::users::user::User;
-use crate::streaming::utils::crypto;
use ::iggy_common::change_password::ChangePassword;
use ::iggy_common::create_user::CreateUser;
-use ::iggy_common::delete_user::DeleteUser;
use ::iggy_common::login_user::LoginUser;
use ::iggy_common::update_permissions::UpdatePermissions;
use ::iggy_common::update_user::UpdateUser;
@@ -76,7 +74,6 @@ async fn get_user(
return Err(CustomError::ResourceNotFound);
};
- // Permission check: only required if user is looking for someone else
if user.id != identity.user_id {
state
.shard
@@ -114,58 +111,23 @@ async fn create_user(
Json(command): Json<CreateUser>,
) -> Result<Json<UserInfoDetails>, CustomError> {
command.validate()?;
- state
- .shard
- .shard()
- .metadata
- .perm_create_user(identity.user_id)?;
-
- let user = state
- .shard
- .shard()
- .create_user(
- &command.username,
- &command.password,
- command.status,
- command.permissions.clone(),
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create user, username: {}",
- command.username
- )
- })?;
-
- let user_id = user.id;
- let response = Json(mapper::map_user(&user));
- {
- let username = command.username.clone();
- let entry_command = EntryCommand::CreateUser(CreateUserWithId {
- user_id,
- command: CreateUser {
- username: command.username.to_owned(),
- password: crypto::hash_password(&command.password),
- status: command.status,
- permissions: command.permissions.clone(),
- },
- });
- let future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create user, username: {}",
- username
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::CreateUser {
+ user_id: identity.user_id,
+ username: command.username,
+ password: command.password,
+ status: command.status,
+ permissions: command.permissions,
+ });
+
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::CreateUserResponse(user) => {
+ let response = mapper::map_user(&user);
+ Ok(Json(response))
+ }
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected CreateUserResponse"),
}
-
- Ok(response)
}
#[debug_handler]
@@ -178,39 +140,19 @@ async fn update_user(
) -> Result<StatusCode, CustomError> {
command.user_id = Identifier::from_str_value(&user_id)?;
command.validate()?;
- state
- .shard
- .shard()
- .metadata
- .perm_update_user(identity.user_id)?;
-
- state
- .shard
- .shard()
- .update_user(&command.user_id, command.username.clone(), command.status)
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to update user, user ID: {user_id}")
- })?;
- {
- let username = command.username.clone();
- let entry_command = EntryCommand::UpdateUser(command);
- let future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update user, username: {}",
- username.unwrap()
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUser {
+ session_user_id: identity.user_id,
+ user_id: command.user_id,
+ username: command.username,
+ status: command.status,
+ });
+
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdateUserResponse(_) => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected UpdateUserResponse"),
}
-
- Ok(StatusCode::NO_CONTENT)
}
#[debug_handler]
@@ -223,45 +165,18 @@ async fn update_permissions(
) -> Result<StatusCode, CustomError> {
command.user_id = Identifier::from_str_value(&user_id)?;
command.validate()?;
- state
- .shard
- .shard()
- .metadata
- .perm_update_permissions(identity.user_id)?;
-
- // Check if target user is root - cannot change root user permissions
- let target_user = state.shard.shard().get_user(&command.user_id)?;
- if target_user.is_root() {
- return Err(CustomError::from(IggyError::CannotChangePermissions(
- target_user.id,
- )));
- }
- state
- .shard
- .shard()
- .update_permissions(&command.user_id, command.permissions.clone())
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to update permissions, user ID: {user_id}")
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissions {
+ session_user_id: identity.user_id,
+ user_id: command.user_id,
+ permissions: command.permissions,
+ });
- {
- let entry_command = EntryCommand::UpdatePermissions(command);
- let future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update permissions, user ID: {user_id}"
- )
- })?;
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::UpdatePermissionsResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected UpdatePermissionsResponse"),
}
-
- Ok(StatusCode::NO_CONTENT)
}
#[debug_handler]
@@ -275,45 +190,18 @@ async fn change_password(
command.user_id = Identifier::from_str_value(&user_id)?;
command.validate()?;
- // Check if user is changing someone else's password
- let target_user = state.shard.shard().get_user(&command.user_id)?;
- if target_user.id != identity.user_id {
- state
- .shard
- .shard()
- .metadata
- .perm_change_password(identity.user_id)?;
- }
-
- state
- .shard
- .shard()
- .change_password(
- &command.user_id,
- &command.current_password,
- &command.new_password,
- )
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to change password, user ID: {user_id}")
- })?;
-
- {
- let entry_command = EntryCommand::ChangePassword(command);
- let future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await.error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply change password, user ID: {user_id}"
- )
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::ChangePassword {
+ session_user_id: identity.user_id,
+ user_id: command.user_id,
+ current_password: command.current_password,
+ new_password: command.new_password,
+ });
+
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::ChangePasswordResponse => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected ChangePasswordResponse"),
}
-
- Ok(StatusCode::NO_CONTENT)
}
#[debug_handler]
@@ -323,38 +211,18 @@ async fn delete_user(
Extension(identity): Extension<Identity>,
Path(user_id): Path<String>,
) -> Result<StatusCode, CustomError> {
- let identifier_user_id = Identifier::from_str_value(&user_id)?;
- state
- .shard
- .shard()
- .metadata
- .perm_delete_user(identity.user_id)?;
+ let user_id = Identifier::from_str_value(&user_id)?;
- state
- .shard
- .shard()
- .delete_user(&identifier_user_id)
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to delete user with ID: {user_id}")
- })?;
+ let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUser {
+ session_user_id: identity.user_id,
+ user_id,
+ });
- {
- let entry_command = EntryCommand::DeleteUser(DeleteUser {
- user_id: identifier_user_id,
- });
- let future = SendWrapper::new(
- state
- .shard
- .shard()
- .state
- .apply(identity.user_id, &entry_command),
- );
- future.await.error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to apply delete user with ID: {user_id}")
- })?;
+ match state.shard.send_to_control_plane(request).await? {
+ ShardResponse::DeletedUser(_) => Ok(StatusCode::NO_CONTENT),
+ ShardResponse::ErrorResponse(err) => Err(err.into()),
+ _ => unreachable!("Expected DeletedUser"),
}
-
- Ok(StatusCode::NO_CONTENT)
}
#[debug_handler]
diff --git a/core/server/src/io/fs_locks.rs b/core/server/src/io/fs_locks.rs
deleted file mode 100644
index d64a8b524..000000000
--- a/core/server/src/io/fs_locks.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use tokio::sync::Mutex;
-
-/// Global filesystem locks to serialize concurrent filesystem operations.
-/// These locks prevent race conditions when multiple concurrent tasks try to
-/// create/delete streams, topics, or partitions on disk simultaneously.
-#[derive(Debug)]
-pub struct FsLocks {
- /// Lock for stream filesystem operations (create/delete stream directories)
- pub stream_lock: Mutex<()>,
- /// Lock for topic filesystem operations (create/delete topic directories)
- pub topic_lock: Mutex<()>,
- /// Lock for partition filesystem operations (create/delete partition directories)
- pub partition_lock: Mutex<()>,
- /// Lock for user filesystem operations (create/delete user directories)
- pub user_lock: Mutex<()>,
-}
-
-impl FsLocks {
- pub fn new() -> Self {
- Self {
- stream_lock: Mutex::new(()),
- topic_lock: Mutex::new(()),
- partition_lock: Mutex::new(()),
- user_lock: Mutex::new(()),
- }
- }
-}
-
-impl Default for FsLocks {
- fn default() -> Self {
- Self::new()
- }
-}
diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs
index 747e5b241..49f06c069 100644
--- a/core/server/src/io/mod.rs
+++ b/core/server/src/io/mod.rs
@@ -15,6 +15,5 @@
// specific language governing permissions and limitations
// under the License.
-pub mod fs_locks;
pub mod fs_utils;
pub mod storage;
diff --git a/core/server/src/metadata/reader.rs b/core/server/src/metadata/reader.rs
index 765711037..128102b8f 100644
--- a/core/server/src/metadata/reader.rs
+++ b/core/server/src/metadata/reader.rs
@@ -19,6 +19,7 @@ use crate::metadata::{
ConsumerGroupId, ConsumerGroupMeta, InnerMetadata, MetadataReadHandle, PartitionId,
PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta,
};
+use crate::shard::transmission::message::{ResolvedPartition, ResolvedTopic};
use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
@@ -628,10 +629,6 @@ impl Metadata {
})
}
- // ==========================================================================
- // Permission checking methods (perm_*)
- // ==========================================================================
-
/// Inheritance: manage_streams → read_streams → read_topics → poll_messages
pub fn perm_poll_messages(
&self,
@@ -1066,10 +1063,6 @@ impl Metadata {
self.perm_poll_messages(user_id, stream_id, topic_id)
}
- // ==========================================================================
- // User permission methods
- // ==========================================================================
-
pub fn perm_get_user(&self, user_id: u32) -> Result<(), IggyError> {
self.perm_read_users(user_id)
}
@@ -1122,10 +1115,6 @@ impl Metadata {
Err(IggyError::Unauthorized)
}
- // ==========================================================================
- // System permission methods
- // ==========================================================================
-
pub fn perm_get_stats(&self, user_id: u32) -> Result<(), IggyError> {
self.perm_get_server_info(user_id)
}
@@ -1149,6 +1138,289 @@ impl Metadata {
Err(IggyError::Unauthorized)
}
+
+ /// Atomically resolve, authorize, and return stream metadata.
+ pub fn query_stream(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ ) -> Result<Option<StreamMeta>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ perm_get_stream_inner(m, user_id, sid)?;
+ Ok(m.streams.get(sid).cloned())
+ })
+ }
+
+ /// Atomically authorize and return all streams.
+ pub fn query_streams(&self, user_id: u32) -> Result<Vec<StreamMeta>, IggyError> {
+ self.with_metadata(|m| {
+ perm_get_streams_inner(m, user_id)?;
+ Ok(m.streams.iter().map(|(_, s)| s.clone()).collect())
+ })
+ }
+
+ /// Atomically resolve, authorize, and return topic metadata.
+ pub fn query_topic(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<Option<TopicMeta>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ let tid = match resolve_topic_id_inner(m, sid, topic_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ perm_get_topic_inner(m, user_id, sid, tid)?;
+ Ok(m.streams.get(sid).and_then(|s| s.topics.get(tid).cloned()))
+ })
+ }
+
+ /// Atomically resolve, authorize, and return all topics for a stream.
+ pub fn query_topics(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ ) -> Result<Option<Vec<TopicMeta>>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ perm_get_topics_inner(m, user_id, sid)?;
+ Ok(m.streams
+ .get(sid)
+ .map(|s| s.topics.iter().map(|(_, t)| t.clone()).collect()))
+ })
+ }
+
+ /// Atomically resolve, authorize, and return consumer group metadata.
+ pub fn query_consumer_group(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ group_id: &Identifier,
+ ) -> Result<Option<ConsumerGroupMeta>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ let tid = match resolve_topic_id_inner(m, sid, topic_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ let gid = match resolve_consumer_group_id_inner(m, sid, tid, group_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ perm_get_consumer_group_inner(m, user_id, sid, tid)?;
+ Ok(m.streams
+ .get(sid)
+ .and_then(|s| s.topics.get(tid))
+ .and_then(|t| t.consumer_groups.get(gid).cloned()))
+ })
+ }
+
+ /// Atomically resolve, authorize, and return all consumer groups for a topic.
+ pub fn query_consumer_groups(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<Option<Vec<ConsumerGroupMeta>>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ let tid = match resolve_topic_id_inner(m, sid, topic_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ perm_get_consumer_group_inner(m, user_id, sid, tid)?;
+ Ok(m.streams
+ .get(sid)
+ .and_then(|s| s.topics.get(tid))
+ .map(|t| t.consumer_groups.iter().map(|(_, cg)| cg.clone()).collect()))
+ })
+ }
+
+ /// Atomically resolve, authorize, and return user metadata.
+ /// Permission check skipped when requesting own data.
+ pub fn query_user(
+ &self,
+ requesting_user_id: u32,
+ target_user_id: &Identifier,
+ ) -> Result<Option<UserMeta>, IggyError> {
+ self.with_metadata(|m| {
+ let uid = match resolve_user_id_inner(m, target_user_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ if uid != requesting_user_id {
+ perm_get_user_inner(m, requesting_user_id)?;
+ }
+ Ok(m.users.get(uid as usize).cloned())
+ })
+ }
+
+ /// Atomically authorize and return all users.
+ pub fn query_users(&self, user_id: u32) -> Result<Vec<UserMeta>, IggyError> {
+ self.with_metadata(|m| {
+ perm_get_users_inner(m, user_id)?;
+ Ok(m.users.iter().map(|(_, u)| u.clone()).collect())
+ })
+ }
+
+ /// Atomically resolve topic and check permission for consumer offset query.
+ /// Returns resolved topic for use with get_consumer_offset.
+ pub fn resolve_for_consumer_offset(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<Option<ResolvedTopic>, IggyError> {
+ self.with_metadata(|m| {
+ let sid = match resolve_stream_id_inner(m, stream_id) {
+ Some(s) => s,
+ None => return Ok(None),
+ };
+ let tid = match resolve_topic_id_inner(m, sid, topic_id) {
+ Some(id) => id,
+ None => return Ok(None),
+ };
+ perm_get_consumer_offset_inner(m, user_id, sid, tid)?;
+ Ok(Some(ResolvedTopic {
+ stream_id: sid,
+ topic_id: tid,
+ }))
+ })
+ }
+
+ /// Atomically resolve topic and check append permission.
+ pub fn resolve_for_append(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<ResolvedTopic, IggyError> {
+ self.with_metadata(|m| {
+ let sid = resolve_stream_id_inner(m, stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
+ let tid = resolve_topic_id_inner(m, sid, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ perm_append_messages_inner(m, user_id, sid, tid)?;
+ Ok(ResolvedTopic {
+ stream_id: sid,
+ topic_id: tid,
+ })
+ })
+ }
+
+ /// Atomically resolve topic and check poll permission.
+ pub fn resolve_for_poll(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<ResolvedTopic, IggyError> {
+ self.with_metadata(|m| {
+ let sid = resolve_stream_id_inner(m, stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
+ let tid = resolve_topic_id_inner(m, sid, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ perm_poll_messages_inner(m, user_id, sid, tid)?;
+ Ok(ResolvedTopic {
+ stream_id: sid,
+ topic_id: tid,
+ })
+ })
+ }
+
+ /// Atomically resolve topic and check store consumer offset permission.
+ pub fn resolve_for_store_consumer_offset(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<ResolvedTopic, IggyError> {
+ self.with_metadata(|m| {
+ let sid = resolve_stream_id_inner(m, stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
+ let tid = resolve_topic_id_inner(m, sid, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ perm_get_consumer_offset_inner(m, user_id, sid, tid)?;
+ Ok(ResolvedTopic {
+ stream_id: sid,
+ topic_id: tid,
+ })
+ })
+ }
+
+ /// Atomically resolve topic and check delete consumer offset permission.
+ pub fn resolve_for_delete_consumer_offset(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<ResolvedTopic, IggyError> {
+ self.with_metadata(|m| {
+ let sid = resolve_stream_id_inner(m, stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
+ let tid = resolve_topic_id_inner(m, sid, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ perm_get_consumer_offset_inner(m, user_id, sid, tid)?;
+ Ok(ResolvedTopic {
+ stream_id: sid,
+ topic_id: tid,
+ })
+ })
+ }
+
+ /// Atomically resolve partition and check delete segments permission.
+ pub fn resolve_for_delete_segments(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: PartitionId,
+ ) -> Result<ResolvedPartition, IggyError> {
+ self.with_metadata(|m| {
+ let sid = resolve_stream_id_inner(m, stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
+ let tid = resolve_topic_id_inner(m, sid, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ let exists = m
+ .streams
+ .get(sid)
+ .and_then(|s| s.topics.get(tid))
+ .and_then(|t| t.partitions.get(partition_id))
+ .is_some();
+ if !exists {
+ return Err(IggyError::PartitionNotFound(
+ partition_id,
+ topic_id.clone(),
+ stream_id.clone(),
+ ));
+ }
+ perm_get_topic_inner(m, user_id, sid, tid)?;
+ Ok(ResolvedPartition {
+ stream_id: sid,
+ topic_id: tid,
+ partition_id,
+ })
+ })
+ }
}
/// Information needed to initialize a LocalPartition.
@@ -1160,3 +1432,290 @@ pub struct PartitionInitInfo {
pub consumer_offsets: Arc<ConsumerOffsets>,
pub consumer_group_offsets: Arc<ConsumerGroupOffsets>,
}
+
+fn resolve_stream_id_inner(m: &InnerMetadata, stream_id: &Identifier) -> Option<StreamId> {
+ match stream_id.kind {
+ IdKind::Numeric => {
+ let sid = stream_id.get_u32_value().ok()? as StreamId;
+ if m.streams.get(sid).is_some() {
+ Some(sid)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = stream_id.get_cow_str_value().ok()?;
+ m.stream_index.get(name.as_ref()).copied()
+ }
+ }
+}
+
+fn resolve_topic_id_inner(
+ m: &InnerMetadata,
+ stream_id: StreamId,
+ topic_id: &Identifier,
+) -> Option<TopicId> {
+ let stream = m.streams.get(stream_id)?;
+ match topic_id.kind {
+ IdKind::Numeric => {
+ let tid = topic_id.get_u32_value().ok()? as TopicId;
+ if stream.topics.get(tid).is_some() {
+ Some(tid)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = topic_id.get_cow_str_value().ok()?;
+ stream.topic_index.get(&Arc::from(name.as_ref())).copied()
+ }
+ }
+}
+
+fn resolve_consumer_group_id_inner(
+ m: &InnerMetadata,
+ stream_id: StreamId,
+ topic_id: TopicId,
+ group_id: &Identifier,
+) -> Option<ConsumerGroupId> {
+ let stream = m.streams.get(stream_id)?;
+ let topic = stream.topics.get(topic_id)?;
+ match group_id.kind {
+ IdKind::Numeric => {
+ let gid = group_id.get_u32_value().ok()? as ConsumerGroupId;
+ if topic.consumer_groups.get(gid).is_some() {
+ Some(gid)
+ } else {
+ None
+ }
+ }
+ IdKind::String => {
+ let name = group_id.get_cow_str_value().ok()?;
+ topic
+ .consumer_group_index
+ .get(&Arc::from(name.as_ref()))
+ .copied()
+ }
+ }
+}
+
+fn resolve_user_id_inner(m: &InnerMetadata, user_id: &Identifier) -> Option<UserId> {
+ match user_id.kind {
+ IdKind::Numeric => Some(user_id.get_u32_value().ok()?),
+ IdKind::String => {
+ let name = user_id.get_cow_str_value().ok()?;
+ m.user_index.get(name.as_ref()).copied()
+ }
+ }
+}
+
+fn perm_get_stream_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+) -> Result<(), IggyError> {
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.manage_streams || global.read_streams)
+ {
+ return Ok(());
+ }
+ if let Some(stream_perm) = m.users_stream_permissions.get(&(user_id, stream_id))
+ && (stream_perm.manage_stream || stream_perm.read_stream)
+ {
+ return Ok(());
+ }
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_get_streams_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> {
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.manage_streams || global.read_streams)
+ {
+ return Ok(());
+ }
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_get_topic_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+ topic_id: TopicId,
+) -> Result<(), IggyError> {
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.read_streams
+ || global.manage_streams
+ || global.manage_topics
+ || global.read_topics)
+ {
+ return Ok(());
+ }
+
+ if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) {
+ if stream_permissions.manage_stream
+ || stream_permissions.read_stream
+ || stream_permissions.manage_topics
+ || stream_permissions.read_topics
+ {
+ return Ok(());
+ }
+
+ if let Some(topics) = &stream_permissions.topics
+ && let Some(topic_permissions) = topics.get(&topic_id)
+ && (topic_permissions.manage_topic || topic_permissions.read_topic)
+ {
+ return Ok(());
+ }
+ }
+
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_get_topics_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+) -> Result<(), IggyError> {
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.read_streams
+ || global.manage_streams
+ || global.manage_topics
+ || global.read_topics)
+ {
+ return Ok(());
+ }
+
+ if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id))
+ && (stream_permissions.manage_stream
+ || stream_permissions.read_stream
+ || stream_permissions.manage_topics
+ || stream_permissions.read_topics)
+ {
+ return Ok(());
+ }
+
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_get_consumer_group_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+ topic_id: TopicId,
+) -> Result<(), IggyError> {
+ perm_get_topic_inner(m, user_id, stream_id, topic_id)
+}
+
+fn perm_get_user_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> {
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.manage_users || global.read_users)
+ {
+ return Ok(());
+ }
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_get_users_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> {
+ perm_get_user_inner(m, user_id)
+}
+
+fn perm_get_consumer_offset_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+ topic_id: TopicId,
+) -> Result<(), IggyError> {
+ if m.users_can_poll_all_streams.contains(&user_id) {
+ return Ok(());
+ }
+
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.read_topics
+ || global.manage_topics
+ || global.read_streams
+ || global.manage_streams)
+ {
+ return Ok(());
+ }
+
+ if m.users_can_poll_stream.contains(&(user_id, stream_id)) {
+ return Ok(());
+ }
+
+ let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else {
+ return Err(IggyError::Unauthorized);
+ };
+
+ if stream_permissions.manage_stream || stream_permissions.read_stream {
+ return Ok(());
+ }
+
+ if stream_permissions.manage_topics || stream_permissions.read_topics {
+ return Ok(());
+ }
+
+ if stream_permissions.poll_messages {
+ return Ok(());
+ }
+
+ if let Some(topics) = &stream_permissions.topics
+ && let Some(topic_permissions) = topics.get(&topic_id)
+ && (topic_permissions.manage_topic
+ || topic_permissions.read_topic
+ || topic_permissions.poll_messages)
+ {
+ return Ok(());
+ }
+
+ Err(IggyError::Unauthorized)
+}
+
+fn perm_poll_messages_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+ topic_id: TopicId,
+) -> Result<(), IggyError> {
+ perm_get_consumer_offset_inner(m, user_id, stream_id, topic_id)
+}
+
+fn perm_append_messages_inner(
+ m: &InnerMetadata,
+ user_id: u32,
+ stream_id: StreamId,
+ topic_id: TopicId,
+) -> Result<(), IggyError> {
+ if m.users_can_send_all_streams.contains(&user_id) {
+ return Ok(());
+ }
+
+ if let Some(global) = m.users_global_permissions.get(&user_id)
+ && (global.manage_streams || global.manage_topics)
+ {
+ return Ok(());
+ }
+
+ if m.users_can_send_stream.contains(&(user_id, stream_id)) {
+ return Ok(());
+ }
+
+ let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else {
+ return Err(IggyError::Unauthorized);
+ };
+
+ if stream_permissions.manage_stream
+ || stream_permissions.manage_topics
+ || stream_permissions.send_messages
+ {
+ return Ok(());
+ }
+
+ if let Some(topics) = &stream_permissions.topics
+ && let Some(topic_permissions) = topics.get(&topic_id)
+ && (topic_permissions.manage_topic || topic_permissions.send_messages)
+ {
+ return Ok(());
+ }
+
+ Err(IggyError::Unauthorized)
+}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index ad8267eac..1454c9f93 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -167,7 +167,6 @@ impl IggyShardBuilder {
metadata_writer: self.metadata_writer.map(RefCell::new),
local_partitions,
pending_partition_inits: RefCell::new(AHashSet::new()),
- fs_locks: Default::default(),
encryptor,
config,
_version: version,
diff --git a/core/server/src/shard/communication.rs b/core/server/src/shard/communication.rs
index 10a7fe59a..654949ece 100644
--- a/core/server/src/shard/communication.rs
+++ b/core/server/src/shard/communication.rs
@@ -20,8 +20,8 @@ use crate::shard::{
transmission::{
connector::ShardConnector,
event::ShardEvent,
- frame::ShardFrame,
- message::{ShardMessage, ShardSendRequestResult},
+ frame::{ShardFrame, ShardResponse},
+ message::{ShardMessage, ShardRequest},
},
};
use futures::future::join_all;
@@ -32,50 +32,45 @@ use std::hash::Hasher as _;
use tracing::{error, info, warn};
impl IggyShard {
- pub async fn send_request_to_shard_or_recoil(
+ /// Sends a control-plane request to shard 0's message pump.
+ pub async fn send_to_control_plane(
&self,
- namespace: Option<&IggyNamespace>,
- message: ShardMessage,
- ) -> Result<ShardSendRequestResult, IggyError> {
- if let Some(ns) = namespace {
- if let Some(shard) = self.find_shard(ns) {
- if shard.id == self.id {
- return Ok(ShardSendRequestResult::Recoil(message));
- }
-
- let response = match shard.send_request(message).await {
- Ok(response) => response,
- Err(err) => {
- error!(
- "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
- shard.id
- );
- return Err(err);
- }
- };
- Ok(ShardSendRequestResult::Response(response))
- } else {
- Err(IggyError::ShardNotFound(
- ns.stream_id(),
- ns.topic_id(),
- ns.partition_id(),
- ))
- }
- } else {
- if self.id == 0 {
- return Ok(ShardSendRequestResult::Recoil(message));
- }
+ request: ShardRequest,
+ ) -> Result<ShardResponse, IggyError> {
+ let shard0 = &self.shards[0];
+ shard0
+ .send_request(ShardMessage::Request(request))
+ .await
+ .map_err(|err| {
+ error!(
+ "{COMPONENT} - failed to send control-plane request to shard 0, error: {err}"
+ );
+ err
+ })
+ }
- let shard0 = &self.shards[0];
- let response = match shard0.send_request(message).await {
- Ok(response) => response,
- Err(err) => {
- error!("{COMPONENT} - failed to send admin request to shard0, error: {err}");
- return Err(err);
- }
- };
- Ok(ShardSendRequestResult::Response(response))
- }
+ /// Sends a data-plane request to the shard owning the partition.
+ pub async fn send_to_data_plane(
+ &self,
+ request: ShardRequest,
+ ) -> Result<ShardResponse, IggyError> {
+ let ns = request
+ .routing
+ .as_ref()
+ .expect("data-plane request requires namespace");
+ let shard = self.find_shard(ns).ok_or_else(|| {
+ IggyError::ShardNotFound(ns.stream_id(), ns.topic_id(), ns.partition_id())
+ })?;
+ shard
+ .send_request(ShardMessage::Request(request))
+ .await
+ .map_err(|err| {
+ error!(
+ "{COMPONENT} - failed to send data-plane request to shard {}, error: {err}",
+ shard.id
+ );
+ err
+ })
}
pub async fn broadcast_event_to_all_shards(&self, event: ShardEvent) -> Result<(), IggyError> {
diff --git a/core/server/src/shard/execution.rs b/core/server/src/shard/execution.rs
new file mode 100644
index 000000000..b035f43c8
--- /dev/null
+++ b/core/server/src/shard/execution.rs
@@ -0,0 +1,705 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::streaming::users::user::User;
+use crate::{
+ shard::{
+ IggyShard,
+ transmission::{
+ event::ShardEvent,
+ frame::{ConsumerGroupResponseData, StreamResponseData, TopicResponseData},
+ message::ResolvedTopic,
+ },
+ },
+ state::{
+ command::EntryCommand,
+ models::{
+ CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId,
+ CreateTopicWithId, CreateUserWithId,
+ },
+ },
+ streaming::polling_consumer::ConsumerGroupId,
+};
+use iggy_common::{
+ Identifier, IggyError, IggyExpiry, Permissions, PersonalAccessToken, UserStatus,
+ change_password::ChangePassword, create_consumer_group::CreateConsumerGroup,
+ create_partitions::CreatePartitions, create_personal_access_token::CreatePersonalAccessToken,
+ create_stream::CreateStream, create_topic::CreateTopic,
+ delete_consumer_group::DeleteConsumerGroup, delete_partitions::DeletePartitions,
+ delete_personal_access_token::DeletePersonalAccessToken, delete_stream::DeleteStream,
+ delete_topic::DeleteTopic, delete_user::DeleteUser, join_consumer_group::JoinConsumerGroup,
+ leave_consumer_group::LeaveConsumerGroup, purge_stream::PurgeStream, purge_topic::PurgeTopic,
+ update_permissions::UpdatePermissions, update_stream::UpdateStream, update_topic::UpdateTopic,
+ update_user::UpdateUser,
+};
+use std::rc::Rc;
+
+pub struct DeleteStreamResult {
+ pub stream_id: usize,
+}
+
+pub struct DeleteTopicResult {
+ pub topic_id: usize,
+}
+
+pub struct CreatePartitionsResult {
+ pub partition_ids: Vec<usize>,
+}
+
+pub struct DeletePartitionsResult {
+ pub partition_ids: Vec<usize>,
+}
+
+pub async fn execute_create_stream(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: CreateStream,
+) -> Result<StreamResponseData, IggyError> {
+ shard.metadata.perm_create_stream(user_id)?;
+
+ let stream_id = shard.create_stream(command.name.clone()).await?;
+
+ // Capture response data from metadata before state apply
+ let response_data = shard.metadata.with_metadata(|m| {
+ let stream = m.streams.get(stream_id).expect("just created");
+ StreamResponseData {
+ id: stream_id as u32,
+ name: stream.name.clone(),
+ created_at: stream.created_at,
+ }
+ });
+
+ shard
+ .state
+ .apply(
+ user_id,
+ &EntryCommand::CreateStream(CreateStreamWithId {
+ stream_id: stream_id as u32,
+ command,
+ }),
+ )
+ .await?;
+
+ Ok(response_data)
+}
+
+pub async fn execute_update_stream(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: UpdateStream,
+) -> Result<(), IggyError> {
+ let stream = shard.resolve_stream(&command.stream_id)?;
+ shard.metadata.perm_update_stream(user_id, stream.id())?;
+
+ shard.update_stream(stream, command.name.clone())?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::UpdateStream(command))
+ .await?;
+
+ Ok(())
+}
+
+pub async fn execute_delete_stream(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: DeleteStream,
+) -> Result<DeleteStreamResult, IggyError> {
+ let stream = shard.resolve_stream(&command.stream_id)?;
+ shard.metadata.perm_delete_stream(user_id, stream.id())?;
+
+ // Capture all topic/partition info BEFORE deletion for broadcast
+ let topics_with_partitions: Vec<(usize, Vec<usize>)> = shard
+ .metadata
+ .get_topic_ids(stream.id())
+ .into_iter()
+ .map(|topic_id| {
+ let partition_ids = shard.metadata.get_partition_ids(stream.id(), topic_id);
+ (topic_id, partition_ids)
+ })
+ .collect();
+
+ let stream_info = shard.delete_stream(stream).await?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeleteStream(command))
+ .await?;
+
+ // Broadcast DeletedPartitions to all shards for each topic's partitions (best-effort)
+ for (topic_id, partition_ids) in topics_with_partitions {
+ if partition_ids.is_empty() {
+ continue;
+ }
+ let event = ShardEvent::DeletedPartitions {
+ stream_id: Identifier::numeric(stream.id() as u32).unwrap(),
+ topic_id: Identifier::numeric(topic_id as u32).unwrap(),
+ partitions_count: partition_ids.len() as u32,
+ partition_ids,
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+ }
+
+ Ok(DeleteStreamResult {
+ stream_id: stream_info.id,
+ })
+}
+
+pub async fn execute_purge_stream(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: PurgeStream,
+) -> Result<(), IggyError> {
+ let stream = shard.resolve_stream(&command.stream_id)?;
+ shard.metadata.perm_purge_stream(user_id, stream.id())?;
+
+ shard.purge_stream(stream).await?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::PurgeStream(command))
+ .await?;
+
+ let event = ShardEvent::PurgedStream {
+ stream_id: Identifier::numeric(stream.id() as u32).unwrap(),
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(())
+}
+
+pub async fn execute_create_topic(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: CreateTopic,
+) -> Result<TopicResponseData, IggyError> {
+ let stream = shard.resolve_stream(&command.stream_id)?;
+ shard.metadata.perm_create_topic(user_id, stream.id())?;
+
+ let topic_id = shard
+ .create_topic(
+ stream,
+ command.name.clone(),
+ command.message_expiry,
+ command.compression_algorithm,
+ command.max_topic_size,
+ command.replication_factor,
+ )
+ .await?;
+
+ let resolved_topic = ResolvedTopic {
+ stream_id: stream.id(),
+ topic_id,
+ };
+ let partition_infos = shard
+ .create_partitions(resolved_topic, command.partitions_count)
+ .await?;
+
+ let response_data = shard.metadata.with_metadata(|m| {
+ let topic = m
+ .streams
+ .get(stream.id())
+ .and_then(|s| s.topics.get(topic_id))
+ .expect("just created");
+ TopicResponseData {
+ id: topic_id as u32,
+ name: topic.name.clone(),
+ created_at: topic.created_at,
+ partitions_count: partition_infos.len() as u32,
+ message_expiry: topic.message_expiry,
+ compression_algorithm: topic.compression_algorithm,
+ max_topic_size: topic.max_topic_size,
+ replication_factor: topic.replication_factor,
+ }
+ });
+
+ shard
+ .state
+ .apply(
+ user_id,
+ &EntryCommand::CreateTopic(CreateTopicWithId {
+ topic_id: topic_id as u32,
+ command,
+ }),
+ )
+ .await?;
+
+ let event = ShardEvent::CreatedPartitions {
+ stream_id: Identifier::numeric(stream.id() as u32).unwrap(),
+ topic_id: Identifier::numeric(topic_id as u32).unwrap(),
+ partitions: partition_infos,
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(response_data)
+}
+
+pub async fn execute_update_topic(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: UpdateTopic,
+) -> Result<(), IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_update_topic(user_id, topic.stream_id, topic.topic_id)?;
+
+ shard.update_topic(
+ topic,
+ command.name.clone(),
+ command.message_expiry,
+ command.compression_algorithm,
+ command.max_topic_size,
+ command.replication_factor,
+ )?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::UpdateTopic(command))
+ .await?;
+
+ Ok(())
+}
+
+pub async fn execute_delete_topic(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: DeleteTopic,
+) -> Result<DeleteTopicResult, IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_delete_topic(user_id, topic.stream_id, topic.topic_id)?;
+
+ // Capture partition_ids BEFORE deletion for broadcast
+ let partition_ids = shard
+ .metadata
+ .get_partition_ids(topic.stream_id, topic.topic_id);
+
+ let topic_info = shard.delete_topic(topic).await?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeleteTopic(command))
+ .await?;
+
+ // Broadcast to all shards to clean up their local_partitions entries (best-effort)
+ let event = ShardEvent::DeletedPartitions {
+ stream_id: Identifier::numeric(topic.stream_id as u32).unwrap(),
+ topic_id: Identifier::numeric(topic.topic_id as u32).unwrap(),
+ partitions_count: partition_ids.len() as u32,
+ partition_ids,
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(DeleteTopicResult {
+ topic_id: topic_info.id,
+ })
+}
+
+pub async fn execute_purge_topic(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: PurgeTopic,
+) -> Result<(), IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_purge_topic(user_id, topic.stream_id, topic.topic_id)?;
+
+ shard.purge_topic(topic).await?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::PurgeTopic(command))
+ .await?;
+
+ let event = ShardEvent::PurgedTopic {
+ stream_id: Identifier::numeric(topic.stream_id as u32).unwrap(),
+ topic_id: Identifier::numeric(topic.topic_id as u32).unwrap(),
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(())
+}
+
+pub async fn execute_create_partitions(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: CreatePartitions,
+) -> Result<CreatePartitionsResult, IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_create_partitions(user_id, topic.stream_id, topic.topic_id)?;
+
+ let partition_infos = shard
+ .create_partitions(topic, command.partitions_count)
+ .await?;
+ let partition_ids = partition_infos.iter().map(|p| p.id).collect::<Vec<_>>();
+
+ let total_partition_count = shard
+ .metadata
+ .partitions_count(topic.stream_id, topic.topic_id) as u32;
+ shard.writer().rebalance_consumer_groups_for_topic(
+ topic.stream_id,
+ topic.topic_id,
+ total_partition_count,
+ );
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::CreatePartitions(command))
+ .await?;
+
+ let event = ShardEvent::CreatedPartitions {
+ stream_id: Identifier::numeric(topic.stream_id as u32).unwrap(),
+ topic_id: Identifier::numeric(topic.topic_id as u32).unwrap(),
+ partitions: partition_infos,
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(CreatePartitionsResult { partition_ids })
+}
+
+pub async fn execute_delete_partitions(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: DeletePartitions,
+) -> Result<DeletePartitionsResult, IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_delete_partitions(user_id, topic.stream_id, topic.topic_id)?;
+
+ let deleted_partition_ids = shard
+ .delete_partitions(topic, command.partitions_count)
+ .await?;
+
+ let remaining_partition_count = shard
+ .metadata
+ .partitions_count(topic.stream_id, topic.topic_id)
+ as u32;
+ shard.writer().rebalance_consumer_groups_for_topic(
+ topic.stream_id,
+ topic.topic_id,
+ remaining_partition_count,
+ );
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeletePartitions(command))
+ .await?;
+
+ let event = ShardEvent::DeletedPartitions {
+ stream_id: Identifier::numeric(topic.stream_id as u32).unwrap(),
+ topic_id: Identifier::numeric(topic.topic_id as u32).unwrap(),
+ partitions_count: deleted_partition_ids.len() as u32,
+ partition_ids: deleted_partition_ids.clone(),
+ };
+ if let Err(e) = shard.broadcast_event_to_all_shards(event).await {
+ tracing::warn!("Broadcast failed: {e}. Shards will sync on restart.");
+ }
+
+ Ok(DeletePartitionsResult {
+ partition_ids: deleted_partition_ids,
+ })
+}
+
+pub async fn execute_create_consumer_group(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: CreateConsumerGroup,
+) -> Result<ConsumerGroupResponseData, IggyError> {
+ let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?;
+ shard
+ .metadata
+ .perm_create_consumer_group(user_id, topic.stream_id, topic.topic_id)?;
+
+ let group_id = shard.create_consumer_group(topic, command.name.clone())?;
+
+ let response_data = shard
+ .metadata
+ .get_consumer_group(topic.stream_id, topic.topic_id, group_id)
+ .map(|cg| ConsumerGroupResponseData {
+ id: group_id as u32,
+ name: cg.name.clone(),
+ partitions_count: cg.partitions.len() as u32,
+ })
+ .expect("just created");
+
+ shard
+ .state
+ .apply(
+ user_id,
+ &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
+ group_id: group_id as u32,
+ command,
+ }),
+ )
+ .await?;
+
+ Ok(response_data)
+}
+
+pub async fn execute_delete_consumer_group(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ command: DeleteConsumerGroup,
+) -> Result<(), IggyError> {
+ let group =
+ shard.resolve_consumer_group(&command.stream_id, &command.topic_id, &command.group_id)?;
+ shard
+ .metadata
+ .perm_delete_consumer_group(user_id, group.stream_id, group.topic_id)?;
+
+ let deleted = shard.delete_consumer_group(group)?;
+
+ let cg_id = ConsumerGroupId(deleted.group_id);
+ shard
+ .delete_consumer_group_offsets(
+ cg_id,
+ group.stream_id,
+ group.topic_id,
+ &deleted.partition_ids,
+ )
+ .await?;
+
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeleteConsumerGroup(command))
+ .await?;
+
+ Ok(())
+}
+
+pub fn execute_join_consumer_group(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ client_id: u32,
+ command: JoinConsumerGroup,
+) -> Result<(), IggyError> {
+ let group =
+ shard.resolve_consumer_group(&command.stream_id, &command.topic_id, &command.group_id)?;
+ shard
+ .metadata
+ .perm_join_consumer_group(user_id, group.stream_id, group.topic_id)?;
+
+ shard.join_consumer_group(client_id, group)?;
+
+ Ok(())
+}
+
+pub fn execute_leave_consumer_group(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ client_id: u32,
+ command: LeaveConsumerGroup,
+) -> Result<(), IggyError> {
+ let group =
+ shard.resolve_consumer_group(&command.stream_id, &command.topic_id, &command.group_id)?;
+ shard
+ .metadata
+ .perm_leave_consumer_group(user_id, group.stream_id, group.topic_id)?;
+
+ shard.leave_consumer_group(client_id, group)?;
+
+ Ok(())
+}
+
+pub async fn execute_create_user(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ username: String,
+ password: String,
+ status: UserStatus,
+ permissions: Option<Permissions>,
+) -> Result<User, IggyError> {
+ shard.metadata.perm_create_user(user_id)?;
+
+ let user = shard.create_user(&username, &password, status, permissions.clone())?;
+
+ let command = iggy_common::create_user::CreateUser {
+ username,
+ password,
+ status,
+ permissions,
+ };
+ shard
+ .state
+ .apply(
+ user_id,
+ &EntryCommand::CreateUser(CreateUserWithId {
+ user_id: user.id,
+ command,
+ }),
+ )
+ .await?;
+
+ Ok(user)
+}
+
+pub async fn execute_delete_user(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ target_user_id: Identifier,
+) -> Result<User, IggyError> {
+ shard.metadata.perm_delete_user(user_id)?;
+
+ let user = shard.delete_user(&target_user_id)?;
+
+ let command = DeleteUser {
+ user_id: target_user_id,
+ };
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeleteUser(command))
+ .await?;
+
+ Ok(user)
+}
+
+pub async fn execute_update_user(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ target_user_id: Identifier,
+ username: Option<String>,
+ status: Option<UserStatus>,
+) -> Result<User, IggyError> {
+ shard.metadata.perm_update_user(user_id)?;
+
+ let user = shard.update_user(&target_user_id, username.clone(), status)?;
+
+ let command = UpdateUser {
+ user_id: target_user_id,
+ username,
+ status,
+ };
+ shard
+ .state
+ .apply(user_id, &EntryCommand::UpdateUser(command))
+ .await?;
+
+ Ok(user)
+}
+
+pub async fn execute_change_password(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ target_user_id: Identifier,
+ current_password: String,
+ new_password: String,
+) -> Result<(), IggyError> {
+ let target_user = shard.get_user(&target_user_id)?;
+ if target_user.id != user_id {
+ shard.metadata.perm_change_password(user_id)?;
+ }
+
+ shard.change_password(&target_user_id, ¤t_password, &new_password)?;
+
+ let command = ChangePassword {
+ user_id: target_user_id,
+ current_password,
+ new_password,
+ };
+ shard
+ .state
+ .apply(user_id, &EntryCommand::ChangePassword(command))
+ .await?;
+
+ Ok(())
+}
+
+pub async fn execute_update_permissions(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ target_user_id: Identifier,
+ permissions: Option<Permissions>,
+) -> Result<(), IggyError> {
+ shard.metadata.perm_update_permissions(user_id)?;
+
+ let target_user = shard.get_user(&target_user_id)?;
+ if target_user.is_root() {
+ return Err(IggyError::CannotChangePermissions(target_user.id));
+ }
+
+ shard.update_permissions(&target_user_id, permissions.clone())?;
+
+ let command = UpdatePermissions {
+ user_id: target_user_id,
+ permissions,
+ };
+ shard
+ .state
+ .apply(user_id, &EntryCommand::UpdatePermissions(command))
+ .await?;
+
+ Ok(())
+}
+
+pub async fn execute_create_personal_access_token(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ name: String,
+ expiry: IggyExpiry,
+) -> Result<(PersonalAccessToken, String), IggyError> {
+ let (personal_access_token, token) =
+ shard.create_personal_access_token(user_id, &name, expiry)?;
+
+ let command = CreatePersonalAccessToken { name, expiry };
+ shard
+ .state
+ .apply(
+ user_id,
+ &EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash {
+ hash: personal_access_token.token.to_string(),
+ command,
+ }),
+ )
+ .await?;
+
+ Ok((personal_access_token, token))
+}
+
+pub async fn execute_delete_personal_access_token(
+ shard: &Rc<IggyShard>,
+ user_id: u32,
+ name: String,
+) -> Result<(), IggyError> {
+ shard.delete_personal_access_token(user_id, &name)?;
+
+ let command = DeletePersonalAccessToken { name };
+ shard
+ .state
+ .apply(user_id, &EntryCommand::DeletePersonalAccessToken(command))
+ .await?;
+
+ Ok(())
+}
diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs
index 53756d5f8..a1f3c37ac 100644
--- a/core/server/src/shard/handlers.rs
+++ b/core/server/src/shard/handlers.rs
@@ -17,9 +17,8 @@
use super::*;
use crate::{
- metadata::{PartitionId, TopicId},
shard::{
- IggyShard,
+ IggyShard, execution,
transmission::{
event::ShardEvent,
frame::ShardResponse,
@@ -32,8 +31,7 @@ use crate::{
},
};
use compio_net::TcpStream;
-use iggy_common::sharding::IggyNamespace;
-use iggy_common::{Identifier, IggyError, SenderKind, TransportProtocol};
+use iggy_common::{IggyError, SenderKind, TransportProtocol, sharding::IggyNamespace};
use nix::sys::stat::SFlag;
use std::os::fd::{FromRawFd, IntoRawFd};
use tracing::info;
@@ -58,16 +56,14 @@ async fn handle_request(
shard: &Rc<IggyShard>,
request: ShardRequest,
) -> Result<ShardResponse, IggyError> {
- let stream_id = request.stream_id;
- let topic_id = request.topic_id;
- let partition_id = request.partition_id;
+ // Data-plane operations extract namespace from routing
+ let namespace = request.routing;
match request.payload {
ShardRequestPayload::SendMessages { batch } => {
let batch = shard.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
- let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
- let namespace = IggyNamespace::new(stream, topic, partition_id);
+ let namespace = namespace.expect("SendMessages requires routing namespace");
shard.ensure_partition(&namespace).await?;
@@ -81,8 +77,7 @@ async fn handle_request(
ShardRequestPayload::PollMessages { args, consumer } => {
let auto_commit = args.auto_commit;
- let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
- let namespace = IggyNamespace::new(stream, topic, partition_id);
+ let namespace = namespace.expect("PollMessages requires routing namespace");
shard.ensure_partition(&namespace).await?;
@@ -101,296 +96,81 @@ async fn handle_request(
Ok(ShardResponse::PollMessages((poll_metadata, batches)))
}
ShardRequestPayload::FlushUnsavedBuffer { fsync } => {
- let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
+ let ns = namespace.expect("FlushUnsavedBuffer requires routing namespace");
shard
- .flush_unsaved_buffer_base(stream, topic, partition_id, fsync)
+ .flush_unsaved_buffer_base(ns.stream_id(), ns.topic_id(), ns.partition_id(), fsync)
.await?;
Ok(ShardResponse::FlushUnsavedBuffer)
}
ShardRequestPayload::DeleteSegments { segments_count } => {
- let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
+ let ns = namespace.expect("DeleteSegments requires routing namespace");
shard
- .delete_segments_base(stream, topic, partition_id, segments_count)
+ .delete_segments(
+ ns.stream_id(),
+ ns.topic_id(),
+ ns.partition_id(),
+ segments_count,
+ )
.await?;
Ok(ShardResponse::DeleteSegments)
}
- ShardRequestPayload::CreatePartitions {
- user_id,
- stream_id,
- topic_id,
- partitions_count,
- } => {
+ ShardRequestPayload::CreatePartitionsRequest { user_id, command } => {
assert_eq!(
shard.id, 0,
- "CreatePartitions should only be handled by shard0"
+ "CreatePartitionsRequest should only be handled by shard0"
);
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- let partition_infos = shard
- .create_partitions(&stream_id, &topic_id, partitions_count)
- .await?;
- let partition_ids = partition_infos.iter().map(|p| p.id).collect::<Vec<_>>();
-
- let event = ShardEvent::CreatedPartitions {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- partitions: partition_infos,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
- let total_partition_count = shard
- .metadata
- .partitions_count(numeric_stream_id, numeric_topic_id)
- as u32;
- shard.writer().rebalance_consumer_groups_for_topic(
- numeric_stream_id,
- numeric_topic_id,
- total_partition_count,
- );
-
- let command = iggy_common::create_partitions::CreatePartitions {
- stream_id,
- topic_id,
- partitions_count,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::CreatePartitions(command),
- )
- .await?;
-
- Ok(ShardResponse::CreatePartitionsResponse(partition_ids))
+ let result = execution::execute_create_partitions(shard, user_id, command).await?;
+ Ok(ShardResponse::CreatePartitionsResponse(
+ result.partition_ids,
+ ))
}
- ShardRequestPayload::DeletePartitions {
- user_id,
- stream_id,
- topic_id,
- partitions_count,
- } => {
+ ShardRequestPayload::DeletePartitionsRequest { user_id, command } => {
assert_eq!(
shard.id, 0,
- "DeletePartitions should only be handled by shard0"
- );
-
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- let deleted_partition_ids = shard
- .delete_partitions(&stream_id, &topic_id, partitions_count)
- .await?;
-
- let event = ShardEvent::DeletedPartitions {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- partitions_count,
- partition_ids: deleted_partition_ids.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
- let remaining_partition_count = shard
- .metadata
- .partitions_count(numeric_stream_id, numeric_topic_id)
- as u32;
- shard.writer().rebalance_consumer_groups_for_topic(
- numeric_stream_id,
- numeric_topic_id,
- remaining_partition_count,
+ "DeletePartitionsRequest should only be handled by shard0"
);
- let command = iggy_common::delete_partitions::DeletePartitions {
- stream_id,
- topic_id,
- partitions_count,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::DeletePartitions(command),
- )
- .await?;
-
+ let result = execution::execute_delete_partitions(shard, user_id, command).await?;
Ok(ShardResponse::DeletePartitionsResponse(
- deleted_partition_ids,
+ result.partition_ids,
))
}
- ShardRequestPayload::CreateStream { user_id, name } => {
- assert_eq!(shard.id, 0, "CreateStream should only be handled by shard0");
-
- // Acquire stream lock to serialize filesystem operations
- let _stream_guard = shard.fs_locks.stream_lock.lock().await;
-
- let created_stream_id = shard.create_stream(name.clone()).await?;
-
- let command = iggy_common::create_stream::CreateStream { name };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::CreateStream(
- crate::state::models::CreateStreamWithId {
- stream_id: created_stream_id as u32,
- command,
- },
- ),
- )
- .await?;
+ ShardRequestPayload::CreateStreamRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "CreateStreamRequest should only be handled by shard0"
+ );
- Ok(ShardResponse::CreateStreamResponse(created_stream_id))
+ let result = execution::execute_create_stream(shard, user_id, command).await?;
+ Ok(ShardResponse::CreateStreamResponse(result))
}
- ShardRequestPayload::CreateTopic {
- user_id,
- stream_id,
- name,
- partitions_count,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- } => {
- assert_eq!(shard.id, 0, "CreateTopic should only be handled by shard0");
-
- // Acquire topic lock to serialize filesystem operations
- let _topic_guard = shard.fs_locks.topic_lock.lock().await;
-
- let topic_id_num = shard
- .create_topic(
- &stream_id,
- name.clone(),
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )
- .await?;
-
- let partition_infos = shard
- .create_partitions(
- &stream_id,
- &Identifier::numeric(topic_id_num as u32).unwrap(),
- partitions_count,
- )
- .await?;
-
- let event = ShardEvent::CreatedPartitions {
- stream_id: stream_id.clone(),
- topic_id: Identifier::numeric(topic_id_num as u32).unwrap(),
- partitions: partition_infos,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let command = iggy_common::create_topic::CreateTopic {
- stream_id,
- partitions_count,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor,
- name,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::CreateTopic(
- crate::state::models::CreateTopicWithId {
- topic_id: topic_id_num as u32,
- command,
- },
- ),
- )
- .await?;
+ ShardRequestPayload::CreateTopicRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "CreateTopicRequest should only be handled by shard0"
+ );
- Ok(ShardResponse::CreateTopicResponse(topic_id_num))
+ let result = execution::execute_create_topic(shard, user_id, command).await?;
+ Ok(ShardResponse::CreateTopicResponse(result))
}
- ShardRequestPayload::UpdateTopic {
- user_id,
- stream_id,
- topic_id,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- } => {
- assert_eq!(shard.id, 0, "UpdateTopic should only be handled by shard0");
-
- shard.update_topic(
- &stream_id,
- &topic_id,
- name.clone(),
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )?;
-
- let command = iggy_common::update_topic::UpdateTopic {
- stream_id,
- topic_id,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor,
- name,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::UpdateTopic(command),
- )
- .await?;
+ ShardRequestPayload::UpdateTopicRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "UpdateTopicRequest should only be handled by shard0"
+ );
+ execution::execute_update_topic(shard, user_id, command).await?;
Ok(ShardResponse::UpdateTopicResponse)
}
- ShardRequestPayload::DeleteTopic {
- user_id,
- stream_id,
- topic_id,
- } => {
- assert_eq!(shard.id, 0, "DeleteTopic should only be handled by shard0");
-
- // Capture numeric IDs and partition_ids BEFORE deletion for broadcast.
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
- let partition_ids = shard
- .metadata
- .get_partition_ids(numeric_stream_id, numeric_topic_id);
-
- let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic_info = shard.delete_topic(&stream_id, &topic_id).await?;
- let topic_id_num = topic_info.id;
-
- // Broadcast to all shards to clean up their local_partitions entries.
- // Use numeric Identifiers since the topic is already deleted from metadata.
- let event = ShardEvent::DeletedPartitions {
- stream_id: Identifier::numeric(numeric_stream_id as u32).unwrap(),
- topic_id: Identifier::numeric(numeric_topic_id as u32).unwrap(),
- partitions_count: partition_ids.len() as u32,
- partition_ids,
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let command = iggy_common::delete_topic::DeleteTopic {
- stream_id,
- topic_id,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::DeleteTopic(command),
- )
- .await?;
+ ShardRequestPayload::DeleteTopicRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "DeleteTopicRequest should only be handled by shard0"
+ );
- Ok(ShardResponse::DeleteTopicResponse(topic_id_num))
+ let result = execution::execute_delete_topic(shard, user_id, command).await?;
+ Ok(ShardResponse::DeleteTopicResponse(result.topic_id))
}
ShardRequestPayload::CreateUser {
user_id: session_user_id,
@@ -400,29 +180,15 @@ async fn handle_request(
permissions,
} => {
assert_eq!(shard.id, 0, "CreateUser should only be handled by shard0");
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard.create_user(&username, &password, status, permissions.clone())?;
-
- let command = iggy_common::create_user::CreateUser {
+ let user = execution::execute_create_user(
+ shard,
+ session_user_id,
username,
password,
status,
permissions,
- };
- shard
- .state
- .apply(
- session_user_id,
- &crate::state::command::EntryCommand::CreateUser(
- crate::state::models::CreateUserWithId {
- user_id: user.id,
- command,
- },
- ),
- )
- .await?;
-
+ )
+ .await?;
Ok(ShardResponse::CreateUserResponse(user))
}
ShardRequestPayload::GetStats { .. } => {
@@ -435,87 +201,26 @@ async fn handle_request(
user_id,
} => {
assert_eq!(shard.id, 0, "DeleteUser should only be handled by shard0");
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard.delete_user(&user_id)?;
-
- let command = iggy_common::delete_user::DeleteUser { user_id };
- shard
- .state
- .apply(
- session_user_id,
- &crate::state::command::EntryCommand::DeleteUser(command),
- )
- .await?;
-
+ let user = execution::execute_delete_user(shard, session_user_id, user_id).await?;
Ok(ShardResponse::DeletedUser(user))
}
- ShardRequestPayload::UpdateStream {
- user_id,
- stream_id,
- name,
- } => {
- assert_eq!(shard.id, 0, "UpdateStream should only be handled by shard0");
-
- shard.update_stream(&stream_id, name.clone())?;
-
- let command = iggy_common::update_stream::UpdateStream { stream_id, name };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::UpdateStream(command),
- )
- .await?;
+ ShardRequestPayload::UpdateStreamRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "UpdateStreamRequest should only be handled by shard0"
+ );
+ execution::execute_update_stream(shard, user_id, command).await?;
Ok(ShardResponse::UpdateStreamResponse)
}
- ShardRequestPayload::DeleteStream { user_id, stream_id } => {
- assert_eq!(shard.id, 0, "DeleteStream should only be handled by shard0");
-
- // Capture numeric stream ID and all topic/partition info BEFORE deletion for broadcast.
- let numeric_stream_id = shard.resolve_stream_id(&stream_id)?;
- let topics_with_partitions: Vec<(TopicId, Vec<PartitionId>)> = shard
- .metadata
- .get_topic_ids(numeric_stream_id)
- .into_iter()
- .map(|topic_id| {
- let partition_ids = shard
- .metadata
- .get_partition_ids(numeric_stream_id, topic_id);
- (topic_id, partition_ids)
- })
- .collect();
-
- let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream_info = shard.delete_stream(&stream_id).await?;
- let stream_id_num = stream_info.id;
-
- // Broadcast DeletedPartitions to all shards for each topic's partitions.
- // Use numeric Identifiers since the stream is already deleted from metadata.
- for (topic_id, partition_ids) in topics_with_partitions {
- if partition_ids.is_empty() {
- continue;
- }
- let event = ShardEvent::DeletedPartitions {
- stream_id: Identifier::numeric(numeric_stream_id as u32).unwrap(),
- topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions_count: partition_ids.len() as u32,
- partition_ids,
- };
- shard.broadcast_event_to_all_shards(event).await?;
- }
-
- let command = iggy_common::delete_stream::DeleteStream { stream_id };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::DeleteStream(command),
- )
- .await?;
+ ShardRequestPayload::DeleteStreamRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "DeleteStreamRequest should only be handled by shard0"
+ );
- Ok(ShardResponse::DeleteStreamResponse(stream_id_num))
+ let result = execution::execute_delete_stream(shard, user_id, command).await?;
+ Ok(ShardResponse::DeleteStreamResponse(result.stream_id))
}
ShardRequestPayload::UpdatePermissions {
session_user_id,
@@ -526,22 +231,8 @@ async fn handle_request(
shard.id, 0,
"UpdatePermissions should only be handled by shard0"
);
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- shard.update_permissions(&user_id, permissions.clone())?;
-
- let command = iggy_common::update_permissions::UpdatePermissions {
- user_id,
- permissions,
- };
- shard
- .state
- .apply(
- session_user_id,
- &crate::state::command::EntryCommand::UpdatePermissions(command),
- )
+ execution::execute_update_permissions(shard, session_user_id, user_id, permissions)
.await?;
-
Ok(ShardResponse::UpdatePermissionsResponse)
}
ShardRequestPayload::ChangePassword {
@@ -554,23 +245,14 @@ async fn handle_request(
shard.id, 0,
"ChangePassword should only be handled by shard0"
);
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- shard.change_password(&user_id, ¤t_password, &new_password)?;
-
- let command = iggy_common::change_password::ChangePassword {
+ execution::execute_change_password(
+ shard,
+ session_user_id,
user_id,
current_password,
new_password,
- };
- shard
- .state
- .apply(
- session_user_id,
- &crate::state::command::EntryCommand::ChangePassword(command),
- )
- .await?;
-
+ )
+ .await?;
Ok(ShardResponse::ChangePasswordResponse)
}
ShardRequestPayload::UpdateUser {
@@ -580,121 +262,53 @@ async fn handle_request(
status,
} => {
assert_eq!(shard.id, 0, "UpdateUser should only be handled by shard0");
-
- let _user_guard = shard.fs_locks.user_lock.lock().await;
- let user = shard.update_user(&user_id, username.clone(), status)?;
-
- let command = iggy_common::update_user::UpdateUser {
- user_id,
- username,
- status,
- };
- shard
- .state
- .apply(
- session_user_id,
- &crate::state::command::EntryCommand::UpdateUser(command),
- )
- .await?;
-
+ let user =
+ execution::execute_update_user(shard, session_user_id, user_id, username, status)
+ .await?;
Ok(ShardResponse::UpdateUserResponse(user))
}
- ShardRequestPayload::CreateConsumerGroup {
- user_id,
- stream_id,
- topic_id,
- name,
- } => {
+ ShardRequestPayload::CreateConsumerGroupRequest { user_id, command } => {
assert_eq!(
shard.id, 0,
- "CreateConsumerGroup should only be handled by shard0"
+ "CreateConsumerGroupRequest should only be handled by shard0"
);
- let cg_id = shard.create_consumer_group(&stream_id, &topic_id, name.clone())?;
-
- let command = iggy_common::create_consumer_group::CreateConsumerGroup {
- stream_id,
- topic_id,
- name,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::CreateConsumerGroup(
- crate::state::models::CreateConsumerGroupWithId {
- group_id: cg_id as u32,
- command,
- },
- ),
- )
- .await?;
-
- Ok(ShardResponse::CreateConsumerGroupResponse(cg_id))
+ let result = execution::execute_create_consumer_group(shard, user_id, command).await?;
+ Ok(ShardResponse::CreateConsumerGroupResponse(result))
}
- ShardRequestPayload::JoinConsumerGroup {
- user_id: _,
+ ShardRequestPayload::JoinConsumerGroupRequest {
+ user_id,
client_id,
- stream_id,
- topic_id,
- group_id,
+ command,
} => {
assert_eq!(
shard.id, 0,
- "JoinConsumerGroup should only be handled by shard0"
+ "JoinConsumerGroupRequest should only be handled by shard0"
);
- shard.join_consumer_group(client_id, &stream_id, &topic_id, &group_id)?;
-
+ execution::execute_join_consumer_group(shard, user_id, client_id, command)?;
Ok(ShardResponse::JoinConsumerGroupResponse)
}
- ShardRequestPayload::LeaveConsumerGroup {
- user_id: _,
+ ShardRequestPayload::LeaveConsumerGroupRequest {
+ user_id,
client_id,
- stream_id,
- topic_id,
- group_id,
+ command,
} => {
assert_eq!(
shard.id, 0,
- "LeaveConsumerGroup should only be handled by shard0"
+ "LeaveConsumerGroupRequest should only be handled by shard0"
);
- shard.leave_consumer_group(client_id, &stream_id, &topic_id, &group_id)?;
-
+ execution::execute_leave_consumer_group(shard, user_id, client_id, command)?;
Ok(ShardResponse::LeaveConsumerGroupResponse)
}
- ShardRequestPayload::DeleteConsumerGroup {
- user_id,
- stream_id,
- topic_id,
- group_id,
- } => {
+ ShardRequestPayload::DeleteConsumerGroupRequest { user_id, command } => {
assert_eq!(
shard.id, 0,
- "DeleteConsumerGroup should only be handled by shard0"
+ "DeleteConsumerGroupRequest should only be handled by shard0"
);
- let cg_meta = shard.delete_consumer_group(&stream_id, &topic_id, &group_id)?;
-
- let cg_id = crate::streaming::polling_consumer::ConsumerGroupId(cg_meta.id);
- shard
- .delete_consumer_group_offsets(cg_id, &stream_id, &topic_id, &cg_meta.partitions)
- .await?;
-
- let command = iggy_common::delete_consumer_group::DeleteConsumerGroup {
- stream_id,
- topic_id,
- group_id,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::DeleteConsumerGroup(command),
- )
- .await?;
-
+ execution::execute_delete_consumer_group(shard, user_id, command).await?;
Ok(ShardResponse::DeleteConsumerGroupResponse)
}
ShardRequestPayload::CreatePersonalAccessToken {
@@ -708,24 +322,8 @@ async fn handle_request(
);
let (personal_access_token, token) =
- shard.create_personal_access_token(user_id, &name, expiry)?;
-
- let command = iggy_common::create_personal_access_token::CreatePersonalAccessToken {
- name,
- expiry,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::CreatePersonalAccessToken(
- crate::state::models::CreatePersonalAccessTokenWithHash {
- hash: personal_access_token.token.to_string(),
- command,
- },
- ),
- )
- .await?;
+ execution::execute_create_personal_access_token(shard, user_id, name, expiry)
+ .await?;
Ok(ShardResponse::CreatePersonalAccessTokenResponse(
personal_access_token,
@@ -738,17 +336,7 @@ async fn handle_request(
"DeletePersonalAccessToken should only be handled by shard0"
);
- shard.delete_personal_access_token(user_id, &name)?;
-
- let command =
- iggy_common::delete_personal_access_token::DeletePersonalAccessToken { name };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::DeletePersonalAccessToken(command),
- )
- .await?;
+ execution::execute_delete_personal_access_token(shard, user_id, name).await?;
Ok(ShardResponse::DeletePersonalAccessTokenResponse)
}
@@ -804,15 +392,11 @@ async fn handle_request(
let batch = shard.maybe_encrypt_messages(initial_data)?;
let messages_count = batch.count();
- // Get numeric IDs for local_partitions lookup
- let (numeric_stream_id, numeric_topic_id) =
- shard.resolve_topic_id(&stream_id, &topic_id)?;
-
- let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
- shard.ensure_partition(&namespace).await?;
+ let ns = namespace.expect("SocketTransfer requires routing namespace");
+ shard.ensure_partition(&ns).await?;
shard
- .append_messages_to_local_partition(&namespace, batch, &shard.config.system)
+ .append_messages_to_local_partition(&ns, batch, &shard.config.system)
.await?;
shard.metrics.increment_messages(messages_count as u64);
@@ -852,54 +436,22 @@ async fn handle_request(
Ok(ShardResponse::SocketTransferResponse)
}
- ShardRequestPayload::PurgeStream { user_id, stream_id } => {
- assert_eq!(shard.id, 0, "PurgeStream should only be handled by shard0");
-
- shard.purge_stream(&stream_id).await?;
-
- let event = ShardEvent::PurgedStream {
- stream_id: stream_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let command = iggy_common::purge_stream::PurgeStream { stream_id };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::PurgeStream(command),
- )
- .await?;
+ ShardRequestPayload::PurgeStreamRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "PurgeStreamRequest should only be handled by shard0"
+ );
+ execution::execute_purge_stream(shard, user_id, command).await?;
Ok(ShardResponse::PurgeStreamResponse)
}
- ShardRequestPayload::PurgeTopic {
- user_id,
- stream_id,
- topic_id,
- } => {
- assert_eq!(shard.id, 0, "PurgeTopic should only be handled by shard0");
-
- shard.purge_topic(&stream_id, &topic_id).await?;
-
- let event = ShardEvent::PurgedTopic {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
- let command = iggy_common::purge_topic::PurgeTopic {
- stream_id,
- topic_id,
- };
- shard
- .state
- .apply(
- user_id,
- &crate::state::command::EntryCommand::PurgeTopic(command),
- )
- .await?;
+ ShardRequestPayload::PurgeTopicRequest { user_id, command } => {
+ assert_eq!(
+ shard.id, 0,
+ "PurgeTopicRequest should only be handled by shard0"
+ );
+ execution::execute_purge_topic(shard, user_id, command).await?;
Ok(ShardResponse::PurgeTopicResponse)
}
}
@@ -941,14 +493,16 @@ pub async fn handle_event(shard: &Rc<IggyShard>, event: ShardEvent) -> Result<()
Ok(())
}
ShardEvent::PurgedStream { stream_id } => {
- shard.purge_stream_bypass_auth(&stream_id).await?;
+ let stream = shard.resolve_stream(&stream_id)?;
+ shard.purge_stream(stream).await?;
Ok(())
}
ShardEvent::PurgedTopic {
stream_id,
topic_id,
} => {
- shard.purge_topic_bypass_auth(&stream_id, &topic_id).await?;
+ let topic = shard.resolve_topic(&stream_id, &topic_id)?;
+ shard.purge_topic(topic).await?;
Ok(())
}
ShardEvent::AddressBound { protocol, address } => {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 7f9318e53..ea4e14cfa 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,31 +17,22 @@
* under the License.
*/
-pub mod builder;
-pub mod system;
-pub mod task_registry;
-pub mod tasks;
-pub mod transmission;
-
-mod communication;
-pub mod handlers;
-
-use ahash::AHashSet;
-pub use communication::calculate_shard_assignment;
-
use self::tasks::{continuous, periodic};
use crate::{
+ bootstrap::load_segments,
configs::server::ServerConfig,
- io::fs_locks::FsLocks,
metadata::{Metadata, MetadataWriter},
shard::{task_registry::TaskRegistry, transmission::frame::ShardFrame},
state::file::FileState,
- streaming::partitions::local_partitions::LocalPartitions,
streaming::{
- clients::client_manager::ClientManager, diagnostics::metrics::Metrics, session::Session,
+ clients::client_manager::ClientManager,
+ diagnostics::metrics::Metrics,
+ partitions::{local_partition::LocalPartition, local_partitions::LocalPartitions},
+ session::Session,
utils::ptr::EternalPtr,
},
};
+use ahash::AHashSet;
use builder::IggyShardBuilder;
use dashmap::DashMap;
use iggy_common::SemanticVersion;
@@ -60,6 +51,18 @@ use std::{
use tracing::{debug, error, info, instrument};
use transmission::connector::{Receiver, ShardConnector, StopReceiver};
+pub mod builder;
+pub mod execution;
+pub mod handlers;
+pub mod system;
+pub mod task_registry;
+pub mod tasks;
+pub mod transmission;
+
+mod communication;
+
+pub use communication::calculate_shard_assignment;
+
pub const COMPONENT: &str = "SHARD";
pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
pub const BROADCAST_TIMEOUT: Duration = Duration::from_secs(500);
@@ -77,7 +80,6 @@ pub struct IggyShard {
pub(crate) shards_table: EternalPtr<DashMap<IggyNamespace, PartitionLocation>>,
pub(crate) state: FileState,
- pub(crate) fs_locks: FsLocks,
pub(crate) encryptor: Option<EncryptorKind>,
pub(crate) config: ServerConfig,
pub(crate) client_manager: ClientManager,
@@ -197,9 +199,6 @@ impl IggyShard {
}
async fn load_segments(&self) -> Result<(), IggyError> {
- use crate::bootstrap::load_segments;
- use crate::streaming::partitions::local_partition::LocalPartition;
-
for shard_entry in self.shards_table.iter() {
let (namespace, location) = shard_entry.pair();
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index 25148ef0e..84c9530f3 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -18,12 +18,10 @@
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload};
use crate::streaming::clients::client_manager::Client;
use crate::streaming::session::Session;
-use iggy_common::{Identifier, TransportProtocol};
+use iggy_common::TransportProtocol;
use std::net::SocketAddr;
use tracing::{error, info, warn};
@@ -59,38 +57,22 @@ impl IggyShard {
}
for (stream_id, topic_id, consumer_group_id) in consumer_groups.into_iter() {
- let request = ShardRequest {
- stream_id: Identifier::numeric(stream_id).unwrap(),
- topic_id: Identifier::numeric(topic_id).unwrap(),
- partition_id: 0,
- payload: ShardRequestPayload::LeaveConsumerGroupMetadataOnly {
+ let request =
+ ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupMetadataOnly {
stream_id: stream_id as usize,
topic_id: topic_id as usize,
group_id: consumer_group_id as usize,
client_id,
- },
- };
+ });
- let message = ShardMessage::Request(request);
- match self.send_request_to_shard_or_recoil(None, message).await {
- Ok(ShardSendRequestResult::Recoil(_)) => {
- // We're on shard 0, do the leave directly
- self.writer().leave_consumer_group(
- stream_id as usize,
- topic_id as usize,
- consumer_group_id as usize,
- client_id,
+ match self.send_to_control_plane(request).await {
+ Ok(ShardResponse::LeaveConsumerGroupMetadataOnlyResponse) => {}
+ Ok(ShardResponse::ErrorResponse(err)) => {
+ warn!(
+ "Failed to leave consumer group {consumer_group_id} for client {client_id} during cleanup: {err}"
);
}
- Ok(ShardSendRequestResult::Response(response)) => match response {
- ShardResponse::LeaveConsumerGroupMetadataOnlyResponse => {}
- ShardResponse::ErrorResponse(err) => {
- warn!(
- "Failed to leave consumer group {consumer_group_id} for client {client_id} during cleanup: {err}"
- );
- }
- _ => {}
- },
+ Ok(_) => {}
Err(err) => {
warn!(
"Failed to send leave consumer group request for client {client_id} during cleanup: {err}"
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index 967eb541a..af1a6683e 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -17,37 +17,44 @@
*/
use super::COMPONENT;
-use crate::metadata::ConsumerGroupMeta;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::{ResolvedConsumerGroup, ResolvedTopic};
use err_trail::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use slab::Slab;
use std::sync::Arc;
+pub struct DeletedConsumerGroup {
+ pub group_id: usize,
+ pub partition_ids: Vec<usize>,
+}
+
impl IggyShard {
pub fn create_consumer_group(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
name: String,
) -> Result<usize, IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
- let partitions_count = self.metadata.partitions_count(stream, topic) as u32;
+ let partitions_count = self.metadata.partitions_count(stream, topic_id) as u32;
let id = self
.writer()
.create_consumer_group(
&self.metadata,
stream,
- topic,
+ topic_id,
Arc::from(name.as_str()),
partitions_count,
)
.map_err(|e| {
if let IggyError::ConsumerGroupNameAlreadyExists(_, _) = &e {
- IggyError::ConsumerGroupNameAlreadyExists(name.clone(), topic_id.clone())
+ IggyError::ConsumerGroupNameAlreadyExists(
+ name.clone(),
+ Identifier::numeric(topic_id as u32).unwrap(),
+ )
} else {
e
}
@@ -58,56 +65,43 @@ impl IggyShard {
pub fn delete_consumer_group(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- group_id: &Identifier,
- ) -> Result<ConsumerGroupMeta, IggyError> {
- let (stream, topic, group) =
- self.resolve_consumer_group_id(stream_id, topic_id, group_id)?;
-
- let cg = self.delete_consumer_group_base(stream, topic, group);
- Ok(cg)
- }
+ group: ResolvedConsumerGroup,
+ ) -> Result<DeletedConsumerGroup, IggyError> {
+ let stream = group.stream_id;
+ let topic = group.topic_id;
+ let group_id = group.group_id;
- fn delete_consumer_group_base(
- &self,
- stream: usize,
- topic: usize,
- group: usize,
- ) -> ConsumerGroupMeta {
- let cg_meta = self
+ let partition_ids = self
.metadata
- .get_consumer_group(stream, topic, group)
- .unwrap_or_else(|| ConsumerGroupMeta {
- id: group,
- name: Arc::from(""),
- partitions: Vec::new(),
- members: Slab::new(),
- });
+ .get_consumer_group(stream, topic, group_id)
+ .map(|cg| cg.partitions.clone())
+ .unwrap_or_default();
self.client_manager
- .delete_consumer_group(stream, topic, group);
+ .delete_consumer_group(stream, topic, group_id);
- self.writer().delete_consumer_group(stream, topic, group);
+ self.writer().delete_consumer_group(stream, topic, group_id);
- cg_meta
+ Ok(DeletedConsumerGroup {
+ group_id,
+ partition_ids,
+ })
}
pub fn join_consumer_group(
&self,
client_id: u32,
- stream_id: &Identifier,
- topic_id: &Identifier,
- group_id: &Identifier,
+ group: ResolvedConsumerGroup,
) -> Result<(), IggyError> {
- let (stream, topic, group) =
- self.resolve_consumer_group_id(stream_id, topic_id, group_id)?;
-
- self.writer()
- .join_consumer_group(stream, topic, group, client_id);
+ self.writer().join_consumer_group(
+ group.stream_id,
+ group.topic_id,
+ group.group_id,
+ client_id,
+ );
self.client_manager
- .join_consumer_group(client_id, stream, topic, group)
+ .join_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id)
.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}",
@@ -121,40 +115,25 @@ impl IggyShard {
pub fn leave_consumer_group(
&self,
client_id: u32,
- stream_id: &Identifier,
- topic_id: &Identifier,
- group_id: &Identifier,
+ group: ResolvedConsumerGroup,
) -> Result<(), IggyError> {
- let (_stream, _topic, _group) =
- self.resolve_consumer_group_id(stream_id, topic_id, group_id)?;
-
- self.leave_consumer_group_base(stream_id, topic_id, group_id, client_id)
- }
-
- pub fn leave_consumer_group_base(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- group_id: &Identifier,
- client_id: u32,
- ) -> Result<(), IggyError> {
- let (stream, topic, group) =
- self.resolve_consumer_group_id(stream_id, topic_id, group_id)?;
-
- let member_id = self
- .writer()
- .leave_consumer_group(stream, topic, group, client_id);
+ let member_id = self.writer().leave_consumer_group(
+ group.stream_id,
+ group.topic_id,
+ group.group_id,
+ client_id,
+ );
if member_id.is_none() {
return Err(IggyError::ConsumerGroupMemberNotFound(
client_id,
- group_id.clone(),
- topic_id.clone(),
+ Identifier::numeric(group.group_id as u32).unwrap(),
+ Identifier::numeric(group.topic_id as u32).unwrap(),
));
}
self.client_manager
- .leave_consumer_group(client_id, stream, topic, group)
+ .leave_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id)
.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}",
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index 961011bf8..89512cf58 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -19,13 +19,16 @@
use super::COMPONENT;
use crate::{
shard::IggyShard,
+ shard::transmission::message::ResolvedTopic,
streaming::{
partitions::consumer_offset::ConsumerOffset,
polling_consumer::{ConsumerGroupId, PollingConsumer},
},
};
use err_trail::ErrContext;
-use iggy_common::{Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError};
+use iggy_common::{
+ Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError, sharding::IggyNamespace,
+};
use std::sync::atomic::Ordering;
impl IggyShard {
@@ -33,16 +36,12 @@ impl IggyShard {
&self,
client_id: u32,
consumer: Consumer,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partition_id: Option<u32>,
offset: u64,
) -> Result<(PollingConsumer, usize), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
- stream_id,
- topic_id,
+ topic,
&consumer,
client_id,
partition_id,
@@ -51,11 +50,32 @@ impl IggyShard {
else {
return Err(IggyError::NotResolvedConsumer(consumer.id));
};
- self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
- self.store_consumer_offset_base(stream, topic, &polling_consumer, partition_id, offset);
- self.persist_consumer_offset_to_disk(stream, topic, &polling_consumer, partition_id)
- .await?;
+ if !self
+ .metadata
+ .partition_exists(topic.stream_id, topic.topic_id, partition_id)
+ {
+ return Err(IggyError::PartitionNotFound(
+ partition_id,
+ Identifier::numeric(topic.topic_id as u32).expect("valid topic id"),
+ Identifier::numeric(topic.stream_id as u32).expect("valid stream id"),
+ ));
+ }
+
+ self.store_consumer_offset_base(
+ topic.stream_id,
+ topic.topic_id,
+ &polling_consumer,
+ partition_id,
+ offset,
+ );
+ self.persist_consumer_offset_to_disk(
+ topic.stream_id,
+ topic.topic_id,
+ &polling_consumer,
+ partition_id,
+ )
+ .await?;
Ok((polling_consumer, partition_id))
}
@@ -63,15 +83,11 @@ impl IggyShard {
&self,
client_id: u32,
consumer: Consumer,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partition_id: Option<u32>,
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
- stream_id,
- topic_id,
+ topic,
&consumer,
client_id,
partition_id,
@@ -80,11 +96,19 @@ impl IggyShard {
else {
return Err(IggyError::NotResolvedConsumer(consumer.id));
};
- self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
- // Get the partition's current offset from stats (messages_count - 1, or 0 if empty)
- use iggy_common::sharding::IggyNamespace;
- let ns = IggyNamespace::new(stream, topic, partition_id);
+ if !self
+ .metadata
+ .partition_exists(topic.stream_id, topic.topic_id, partition_id)
+ {
+ return Err(IggyError::PartitionNotFound(
+ partition_id,
+ Identifier::numeric(topic.topic_id as u32).expect("valid topic id"),
+ Identifier::numeric(topic.stream_id as u32).expect("valid stream id"),
+ ));
+ }
+
+ let ns = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id);
let partition_current_offset = self
.metadata
.get_partition_stats(&ns)
@@ -96,9 +120,11 @@ impl IggyShard {
let offset = match polling_consumer {
PollingConsumer::Consumer(id, _) => {
- let offsets =
- self.metadata
- .get_partition_consumer_offsets(stream, topic, partition_id);
+ let offsets = self.metadata.get_partition_consumer_offsets(
+ topic.stream_id,
+ topic.topic_id,
+ partition_id,
+ );
offsets.and_then(|co| {
let guard = co.pin();
guard.get(&id).map(|item| ConsumerOffsetInfo {
@@ -109,9 +135,11 @@ impl IggyShard {
})
}
PollingConsumer::ConsumerGroup(consumer_group_id, _) => {
- let offsets =
- self.metadata
- .get_partition_consumer_group_offsets(stream, topic, partition_id);
+ let offsets = self.metadata.get_partition_consumer_group_offsets(
+ topic.stream_id,
+ topic.topic_id,
+ partition_id,
+ );
offsets.and_then(|co| {
let guard = co.pin();
guard
@@ -131,15 +159,11 @@ impl IggyShard {
&self,
client_id: u32,
consumer: Consumer,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partition_id: Option<u32>,
) -> Result<(PollingConsumer, usize), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id(
- stream_id,
- topic_id,
+ topic,
&consumer,
client_id,
partition_id,
@@ -148,10 +172,24 @@ impl IggyShard {
else {
return Err(IggyError::NotResolvedConsumer(consumer.id));
};
- self.ensure_partition_exists(stream_id, topic_id, partition_id)?;
- let path =
- self.delete_consumer_offset_base(stream, topic, &polling_consumer, partition_id)?;
+ if !self
+ .metadata
+ .partition_exists(topic.stream_id, topic.topic_id, partition_id)
+ {
+ return Err(IggyError::PartitionNotFound(
+ partition_id,
+ Identifier::numeric(topic.topic_id as u32).expect("valid topic id"),
+ Identifier::numeric(topic.stream_id as u32).expect("valid stream id"),
+ ));
+ }
+
+ let path = self.delete_consumer_offset_base(
+ topic.stream_id,
+ topic.topic_id,
+ &polling_consumer,
+ partition_id,
+ )?;
self.delete_consumer_offset_from_disk(&path).await?;
Ok((polling_consumer, partition_id))
}
@@ -159,16 +197,16 @@ impl IggyShard {
pub async fn delete_consumer_group_offsets(
&self,
cg_id: ConsumerGroupId,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ stream_id: usize,
+ topic_id: usize,
partition_ids: &[usize],
) -> Result<(), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
for &partition_id in partition_ids {
- let offsets =
- self.metadata
- .get_partition_consumer_group_offsets(stream, topic, partition_id);
+ let offsets = self.metadata.get_partition_consumer_group_offsets(
+ stream_id,
+ topic_id,
+ partition_id,
+ );
let Some(offsets) = offsets else {
continue;
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index cc2b3a75c..b538e5246 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -21,7 +21,7 @@ use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+ ResolvedPartition, ResolvedTopic, ShardRequest, ShardRequestPayload,
};
use crate::streaming::partitions::journal::Journal;
use crate::streaming::polling_consumer::PollingConsumer;
@@ -37,99 +37,61 @@ use std::sync::atomic::Ordering;
use tracing::error;
impl IggyShard {
+ /// Appends messages to partition. Permission must be checked by caller via
+ /// `resolve_topic_for_append()` before calling this method.
pub async fn append_messages(
&self,
- user_id: u32,
- stream_id: Identifier,
- topic_id: Identifier,
- partition_id: usize,
+ partition: ResolvedPartition,
batch: IggyMessagesBatchMut,
) -> Result<(), IggyError> {
- let (stream, topic, _) = self.resolve_partition_id(&stream_id, &topic_id, partition_id)?;
+ if batch.count() == 0 {
+ return Ok(());
+ }
- self.metadata
- .perm_append_messages(user_id, stream, topic)
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}", user_id, stream as u32, topic as u32)
- })?;
+ let namespace = IggyNamespace::new(
+ partition.stream_id,
+ partition.topic_id,
+ partition.partition_id,
+ );
- if batch.count() == 0 {
+ // Fast path: if partition exists locally, append directly without channel routing
+ let is_local = self.local_partitions.borrow().contains(&namespace);
+ if is_local {
+ let batch = self.maybe_encrypt_messages(batch)?;
+ let messages_count = batch.count();
+
+ self.append_messages_to_local_partition(&namespace, batch, &self.config.system)
+ .await?;
+
+ self.metrics.increment_messages(messages_count as u64);
return Ok(());
}
- // TODO(tungtose): DRY this code
- let namespace = IggyNamespace::new(stream, topic, partition_id);
+ // Slow path: route through data-plane channel to target shard
let payload = ShardRequestPayload::SendMessages { batch };
- let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
- let message = ShardMessage::Request(request);
- match self
- .send_request_to_shard_or_recoil(Some(&namespace), message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest {
- stream_id: _,
- topic_id: _,
- partition_id,
- payload,
- }) = message
- && let ShardRequestPayload::SendMessages { batch } = payload
- {
- let batch = self.maybe_encrypt_messages(batch)?;
- let messages_count = batch.count();
-
- let namespace = IggyNamespace::new(stream, topic, partition_id);
- self.ensure_partition(&namespace).await?;
-
- self.append_messages_to_local_partition(&namespace, batch, &self.config.system)
- .await?;
-
- self.metrics.increment_messages(messages_count as u64);
- Ok(())
- } else {
- unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::SendMessages => Ok(()),
- ShardResponse::ErrorResponse(err) => Err(err),
- _ => unreachable!(
- "Expected a SendMessages response inside of SendMessages handler, impossible state"
- ),
- },
- }?;
+ let request = ShardRequest::data_plane(namespace, payload);
+
+ match self.send_to_data_plane(request).await? {
+ ShardResponse::SendMessages => {}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected SendMessages response"),
+ }
Ok(())
}
- #[allow(clippy::too_many_arguments)]
+ /// Polls messages from partition. Permission must be checked by caller via
+ /// `resolve_topic_for_poll()` before calling this method.
pub async fn poll_messages(
&self,
client_id: u32,
- user_id: u32,
- stream_id: Identifier,
- topic_id: Identifier,
+ topic: ResolvedTopic,
consumer: Consumer,
maybe_partition_id: Option<u32>,
args: PollingArgs,
) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> {
- let (stream, topic) = self.resolve_topic_id(&stream_id, &topic_id)?;
-
- self.metadata
- .perm_poll_messages(user_id, stream, topic)
- .error(|e: &IggyError| format!(
- "{COMPONENT} (error: {e}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
- user_id,
- stream_id,
- topic
- ))?;
-
- // Resolve partition ID
let Some((consumer, partition_id)) = self.resolve_consumer_with_partition_id(
- &stream_id,
- &topic_id,
+ topic,
&consumer,
client_id,
maybe_partition_id,
@@ -139,71 +101,55 @@ impl IggyShard {
return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
};
- self.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
+ let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id);
+
+ // Fast path: if partition exists locally, poll directly without channel routing
+ if self.local_partitions.borrow().contains(&namespace) {
+ if args.count == 0 {
+ let current_offset = self
+ .local_partitions
+ .borrow()
+ .get(&namespace)
+ .map(|data| data.offset.load(Ordering::Relaxed))
+ .unwrap_or(0);
+ return Ok((
+ IggyPollMetadata::new(partition_id as u32, current_offset),
+ IggyMessagesBatchSet::empty(),
+ ));
+ }
- let namespace = IggyNamespace::new(stream, topic, partition_id);
+ let auto_commit = args.auto_commit;
- if args.count == 0 {
- let current_offset = self
- .local_partitions
- .borrow()
- .get(&namespace)
- .map(|data| data.offset.load(Ordering::Relaxed))
- .unwrap_or(0);
- return Ok((
- IggyPollMetadata::new(partition_id as u32, current_offset),
- IggyMessagesBatchSet::empty(),
- ));
+ let (poll_metadata, batches) = self
+ .poll_messages_from_local_partition(&namespace, consumer, args)
+ .await?;
+
+ if auto_commit && !batches.is_empty() {
+ let offset = batches
+ .last_offset()
+ .expect("Batch set should have at least one batch");
+ self.auto_commit_consumer_offset_from_local_partition(&namespace, consumer, offset)
+ .await?;
+ }
+
+ let batches = if let Some(encryptor) = &self.encryptor {
+ self.decrypt_messages(batches, encryptor).await?
+ } else {
+ batches
+ };
+
+ return Ok((poll_metadata, batches));
}
- // Offset validation is done by the owning shard after routing
+ // Slow path: route through data-plane channel to target shard
let payload = ShardRequestPayload::PollMessages { consumer, args };
- let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
- let message = ShardMessage::Request(request);
- let (metadata, batch) = match self
- .send_request_to_shard_or_recoil(Some(&namespace), message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest {
- partition_id: _,
- payload,
- ..
- }) = message
- && let ShardRequestPayload::PollMessages { consumer, args } = payload
- {
- self.ensure_partition(&namespace).await?;
-
- let auto_commit = args.auto_commit;
-
- let (poll_metadata, batches) = self
- .poll_messages_from_local_partition(&namespace, consumer, args)
- .await?;
-
- if auto_commit && !batches.is_empty() {
- let offset = batches
- .last_offset()
- .expect("Batch set should have at least one batch");
- self.auto_commit_consumer_offset_from_local_partition(
- &namespace, consumer, offset,
- )
- .await?;
- }
- Ok((poll_metadata, batches))
- } else {
- unreachable!(
- "Expected a PollMessages request inside of PollMessages handler, impossible state"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::PollMessages(result) => Ok(result),
- ShardResponse::ErrorResponse(err) => Err(err),
- _ => unreachable!(
- "Expected a SendMessages response inside of SendMessages handler, impossible state"
- ),
- },
- }?;
+ let request = ShardRequest::data_plane(namespace, payload);
+
+ let (metadata, batch) = match self.send_to_data_plane(request).await? {
+ ShardResponse::PollMessages(result) => result,
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!("Expected PollMessages response"),
+ };
let batch = if let Some(encryptor) = &self.encryptor {
self.decrypt_messages(batch, encryptor).await?
@@ -217,55 +163,28 @@ impl IggyShard {
pub async fn flush_unsaved_buffer(
&self,
user_id: u32,
- stream_id: Identifier,
- topic_id: Identifier,
- partition_id: usize,
+ partition: ResolvedPartition,
fsync: bool,
) -> Result<(), IggyError> {
- let (stream, topic, _) = self.resolve_partition_id(&stream_id, &topic_id, partition_id)?;
-
self.metadata
- .perm_append_messages(user_id, stream, topic)
+ .perm_append_messages(user_id, partition.stream_id, partition.topic_id)
.error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, stream as u32, topic as u32)
+ format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, partition.stream_id as u32, partition.topic_id as u32)
})?;
- let namespace = IggyNamespace::new(stream, topic, partition_id);
+ let namespace = IggyNamespace::new(
+ partition.stream_id,
+ partition.topic_id,
+ partition.partition_id,
+ );
let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync };
- let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
- let message = ShardMessage::Request(request);
- match self
- .send_request_to_shard_or_recoil(Some(&namespace), message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest {
- partition_id,
- payload,
- ..
- }) = message
- && let ShardRequestPayload::FlushUnsavedBuffer { fsync } = payload
- {
- let namespace = IggyNamespace::new(stream, topic, partition_id);
- self.flush_unsaved_buffer_from_local_partitions(&namespace, fsync)
- .await?;
- Ok(())
- } else {
- unreachable!(
- "Expected a FlushUnsavedBuffer request inside of FlushUnsavedBuffer handler, impossible state"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::FlushUnsavedBuffer => Ok(()),
- ShardResponse::ErrorResponse(err) => Err(err),
- _ => unreachable!(
- "Expected a FlushUnsavedBuffer response inside of FlushUnsavedBuffer handler, impossible state"
- ),
- },
- }?;
+ let request = ShardRequest::data_plane(namespace, payload);
- Ok(())
+ match self.send_to_data_plane(request).await? {
+ ShardResponse::FlushUnsavedBuffer => Ok(()),
+ ShardResponse::ErrorResponse(err) => Err(err),
+ _ => unreachable!("Expected FlushUnsavedBuffer response"),
+ }
}
pub(crate) async fn flush_unsaved_buffer_base(
diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs
index 8b32dce6c..b0d0d8472 100644
--- a/core/server/src/shard/system/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -20,6 +20,7 @@ use crate::metadata::PartitionMeta;
use crate::shard::IggyShard;
use crate::shard::calculate_shard_assignment;
use crate::shard::transmission::event::PartitionInfo;
+use crate::shard::transmission::message::ResolvedTopic;
use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use crate::streaming::partitions::local_partition::LocalPartition;
@@ -44,23 +45,23 @@ const PARTITION_INIT_TIMEOUT: Duration = Duration::from_secs(5);
impl IggyShard {
pub async fn create_partitions(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partitions_count: u32,
) -> Result<Vec<PartitionInfo>, IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
let created_at = IggyTimestamp::now();
let shards_count = self.get_available_shards_count();
let parent_stats = self
.metadata
- .get_topic_stats(stream, topic)
+ .get_topic_stats(stream, topic_id)
.expect("Parent topic stats must exist");
let count_before = self
.metadata
- .get_partitions_count(stream, topic)
+ .get_partitions_count(stream, topic_id)
.unwrap_or(0);
let partition_ids: Vec<usize> =
(count_before..count_before + partitions_count as usize).collect();
@@ -70,7 +71,7 @@ impl IggyShard {
.collect();
for info in &partition_infos {
- create_partition_file_hierarchy(stream, topic, info.id, &self.config.system).await?;
+ create_partition_file_hierarchy(stream, topic_id, info.id, &self.config.system).await?;
}
let metas: Vec<PartitionMeta> = (0..partitions_count)
@@ -86,7 +87,7 @@ impl IggyShard {
let assigned_ids = self
.writer()
- .add_partitions(&self.metadata, stream, topic, metas);
+ .add_partitions(&self.metadata, stream, topic_id, metas);
debug_assert_eq!(
assigned_ids, partition_ids,
"Partition IDs mismatch: expected {:?}, got {:?}",
@@ -98,7 +99,7 @@ impl IggyShard {
for info in &partition_infos {
let partition_id = info.id;
- let ns = IggyNamespace::new(stream, topic, partition_id);
+ let ns = IggyNamespace::new(stream, topic_id, partition_id);
let shard_id = ShardId::new(calculate_shard_assignment(&ns, shards_count));
let is_current_shard = self.id == *shard_id;
// TODO(hubcio): LocalIdx(0) is wrong.. When IggyPartitions is integrated into
@@ -306,15 +307,15 @@ impl IggyShard {
pub async fn delete_partitions(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partitions_count: u32,
) -> Result<Vec<usize>, IggyError> {
- self.ensure_partitions_exist(stream_id, topic_id, partitions_count)?;
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
+ self.validate_partitions_count(topic, partitions_count)?;
- let all_partition_ids = self.metadata.get_partition_ids(stream, topic);
+ let all_partition_ids = self.metadata.get_partition_ids(stream, topic_id);
let partitions_to_delete: Vec<usize> = all_partition_ids
.into_iter()
@@ -322,7 +323,7 @@ impl IggyShard {
.take(partitions_count as usize)
.collect();
- let topic_stats = self.metadata.get_topic_stats(stream, topic);
+ let topic_stats = self.metadata.get_topic_stats(stream, topic_id);
let mut total_messages_count: u64 = 0;
let mut total_segments_count: u32 = 0;
@@ -331,7 +332,7 @@ impl IggyShard {
for partition_id in &partitions_to_delete {
if let Some(stats) =
self.metadata
- .get_partition_stats_by_ids(stream, topic, *partition_id)
+ .get_partition_stats_by_ids(stream, topic_id, *partition_id)
{
total_segments_count += stats.segments_count_inconsistent();
total_messages_count += stats.messages_count_inconsistent();
@@ -340,16 +341,16 @@ impl IggyShard {
}
self.writer()
- .delete_partitions(stream, topic, partitions_to_delete.len() as u32);
+ .delete_partitions(stream, topic_id, partitions_to_delete.len() as u32);
for partition_id in &partitions_to_delete {
- let ns = IggyNamespace::new(stream, topic, *partition_id);
+ let ns = IggyNamespace::new(stream, topic_id, *partition_id);
self.remove_shard_table_record(&ns);
self.local_partitions.borrow_mut().remove(&ns);
}
for partition_id in &partitions_to_delete {
- self.delete_partition_dir(stream, topic, *partition_id)
+ self.delete_partition_dir(stream, topic_id, *partition_id)
.await?;
}
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index 757d82977..46492d33a 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -71,36 +71,24 @@ impl IggyShard {
let (personal_access_token, token) =
PersonalAccessToken::new(user_id, name, IggyTimestamp::now(), expiry);
- self.create_personal_access_token_base(personal_access_token.clone())?;
- Ok((personal_access_token, token))
- }
- fn create_personal_access_token_base(
- &self,
- personal_access_token: PersonalAccessToken,
- ) -> Result<(), IggyError> {
- let user_id = personal_access_token.user_id;
- let name = personal_access_token.name.clone();
-
- if self.metadata.user_has_pat_with_name(user_id, &name) {
- error!("Personal access token: {name} for user with ID: {user_id} already exists.");
+ let pat_name = personal_access_token.name.clone();
+ if self.metadata.user_has_pat_with_name(user_id, &pat_name) {
+ error!("Personal access token: {pat_name} for user with ID: {user_id} already exists.");
return Err(IggyError::PersonalAccessTokenAlreadyExists(
- name.to_string(),
+ pat_name.to_string(),
user_id,
));
}
self.writer()
- .add_personal_access_token(user_id, personal_access_token);
- info!("Created personal access token: {name} for user with ID: {user_id}.");
- Ok(())
- }
+ .add_personal_access_token(user_id, personal_access_token.clone());
+ info!("Created personal access token: {pat_name} for user with ID: {user_id}.");
- pub fn delete_personal_access_token(&self, user_id: u32, name: &str) -> Result<(), IggyError> {
- self.delete_personal_access_token_base(user_id, name)
+ Ok((personal_access_token, token))
}
- fn delete_personal_access_token_base(&self, user_id: u32, name: &str) -> Result<(), IggyError> {
+ pub fn delete_personal_access_token(&self, user_id: u32, name: &str) -> Result<(), IggyError> {
let token_hash =
self.metadata
.find_pat_token_hash_by_name(user_id, name)
diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs
index fb4a68496..b7226b69d 100644
--- a/core/server/src/shard/system/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -21,7 +21,7 @@ use iggy_common::IggyError;
use iggy_common::sharding::IggyNamespace;
impl IggyShard {
- pub(crate) async fn delete_segments_base(
+ pub(crate) async fn delete_segments(
&self,
stream: usize,
topic: usize,
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 380ab8b1b..afccda732 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -18,11 +18,12 @@
use crate::metadata::StreamMeta;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic};
use crate::streaming::streams::storage::{create_stream_file_hierarchy, delete_stream_directory};
-use bytes::{BufMut, BytesMut};
use iggy_common::sharding::IggyNamespace;
-use iggy_common::{Identifier, IggyError, IggyTimestamp};
+use iggy_common::{IggyError, IggyTimestamp};
use std::sync::Arc;
+
/// Info returned when a stream is deleted - contains what callers need for logging/events.
pub struct DeletedStreamInfo {
pub id: usize,
@@ -52,30 +53,29 @@ impl IggyShard {
Ok(stream_id)
}
- pub fn update_stream(&self, stream_id: &Identifier, name: String) -> Result<(), IggyError> {
- let stream_id = self.resolve_stream_id(stream_id)?;
+ pub fn update_stream(&self, stream: ResolvedStream, name: String) -> Result<(), IggyError> {
self.writer()
- .try_update_stream(&self.metadata, stream_id, Arc::from(name.as_str()))
+ .try_update_stream(&self.metadata, stream.id(), Arc::from(name.as_str()))
}
- fn delete_stream_base(&self, stream_id: usize) -> DeletedStreamInfo {
- let (stream_name, stats, topics_count, partitions_count, namespaces) =
+ pub async fn delete_stream(
+ &self,
+ stream: ResolvedStream,
+ ) -> Result<DeletedStreamInfo, IggyError> {
+ let stream_id = stream.id();
+
+ let (topics_with_partitions, stream_name, stats, topics_count, partitions_count) =
self.metadata.with_metadata(|m| {
let stream_meta = m
.streams
.get(stream_id)
.expect("Stream metadata must exist");
- let namespaces: Vec<_> = stream_meta
+ let twp: Vec<_> = stream_meta
.topics
.iter()
- .flat_map(|(topic_id, topic)| {
- topic
- .partitions
- .iter()
- .enumerate()
- .map(move |(partition_id, _)| {
- IggyNamespace::new(stream_id, topic_id, partition_id)
- })
+ .map(|(topic_id, topic)| {
+ let partition_ids: Vec<usize> = (0..topic.partitions.len()).collect();
+ (topic_id, partition_ids)
})
.collect();
let partitions_count: usize = stream_meta
@@ -84,15 +84,23 @@ impl IggyShard {
.map(|(_, t)| t.partitions.len())
.sum();
(
+ twp,
stream_meta.name.to_string(),
stream_meta.stats.clone(),
stream_meta.topics.len(),
partitions_count,
- namespaces,
)
});
{
+ let namespaces: Vec<_> = topics_with_partitions
+ .iter()
+ .flat_map(|(topic_id, partition_ids)| {
+ partition_ids
+ .iter()
+ .map(|&partition_id| IggyNamespace::new(stream_id, *topic_id, partition_id))
+ })
+ .collect();
let mut partitions = self.local_partitions.borrow_mut();
for ns in namespaces {
partitions.remove(&ns);
@@ -109,42 +117,20 @@ impl IggyShard {
self.writer().delete_stream(stream_id);
- DeletedStreamInfo {
+ let stream_info = DeletedStreamInfo {
id: stream_id,
name: stream_name,
- }
- }
-
- pub async fn delete_stream(&self, id: &Identifier) -> Result<DeletedStreamInfo, IggyError> {
- let stream = self.resolve_stream_id(id)?;
-
- let topics_with_partitions = self.metadata.with_metadata(|m| {
- m.streams
- .get(stream)
- .map(|stream_meta| {
- stream_meta
- .topics
- .iter()
- .map(|(topic_id, topic)| {
- let partition_ids: Vec<usize> = (0..topic.partitions.len()).collect();
- (topic_id, partition_ids)
- })
- .collect::<Vec<_>>()
- })
- .unwrap_or_default()
- });
-
- let stream_info = self.delete_stream_base(stream);
+ };
self.client_manager
- .delete_consumer_groups_for_stream(stream);
+ .delete_consumer_groups_for_stream(stream_id);
let namespaces_to_remove: Vec<_> = self
.shards_table
.iter()
.filter_map(|entry| {
let (ns, _) = entry.pair();
- if ns.stream_id() == stream {
+ if ns.stream_id() == stream_id {
Some(*ns)
} else {
None
@@ -156,161 +142,22 @@ impl IggyShard {
self.remove_shard_table_record(&ns);
}
- delete_stream_directory(stream, &topics_with_partitions, &self.config.system).await?;
+ delete_stream_directory(stream_id, &topics_with_partitions, &self.config.system).await?;
Ok(stream_info)
}
- pub async fn purge_stream(&self, stream_id: &Identifier) -> Result<(), IggyError> {
- let stream = self.resolve_stream_id(stream_id)?;
- self.purge_stream_base(stream).await
- }
-
- pub async fn purge_stream_bypass_auth(&self, stream_id: &Identifier) -> Result<(), IggyError> {
- let stream = self.resolve_stream_id(stream_id)?;
- self.purge_stream_base(stream).await
- }
-
- async fn purge_stream_base(&self, stream_id: usize) -> Result<(), IggyError> {
+ pub async fn purge_stream(&self, stream: ResolvedStream) -> Result<(), IggyError> {
+ let stream_id = stream.id();
let topic_ids = self.metadata.get_topic_ids(stream_id);
for topic_id in topic_ids {
- self.purge_topic_base(stream_id, topic_id).await?;
+ let topic = ResolvedTopic {
+ stream_id,
+ topic_id,
+ };
+ self.purge_topic(topic).await?;
}
Ok(())
}
-
- pub fn get_stream_from_metadata(&self, stream_id: usize) -> bytes::Bytes {
- self.metadata.with_metadata(|metadata| {
- let Some(stream_meta) = metadata.streams.get(stream_id) else {
- return bytes::Bytes::new();
- };
-
- let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect();
- topic_ids.sort_unstable();
-
- let (total_size, total_messages) = {
- let mut size = 0u64;
- let mut messages = 0u64;
- for &topic_id in &topic_ids {
- if let Some(topic) = stream_meta.topics.get(topic_id) {
- for partition_id in 0..topic.partitions.len() {
- let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
- if let Some(stats) = metadata
- .streams
- .get(ns.stream_id())
- .and_then(|s| s.topics.get(ns.topic_id()))
- .and_then(|t| t.partitions.get(ns.partition_id()))
- .map(|p| p.stats.clone())
- {
- size += stats.size_bytes_inconsistent();
- messages += stats.messages_count_inconsistent();
- }
- }
- }
- }
- (size, messages)
- };
-
- let mut bytes = BytesMut::new();
-
- bytes.put_u32_le(stream_meta.id as u32);
- bytes.put_u64_le(stream_meta.created_at.into());
- bytes.put_u32_le(topic_ids.len() as u32);
- bytes.put_u64_le(total_size);
- bytes.put_u64_le(total_messages);
- bytes.put_u8(stream_meta.name.len() as u8);
- bytes.put_slice(stream_meta.name.as_bytes());
-
- for &topic_id in &topic_ids {
- if let Some(topic_meta) = stream_meta.topics.get(topic_id) {
- let partition_ids: Vec<_> = (0..topic_meta.partitions.len()).collect();
-
- let (topic_size, topic_messages) = {
- let mut size = 0u64;
- let mut messages = 0u64;
- for &partition_id in &partition_ids {
- let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
- if let Some(stats) = metadata
- .streams
- .get(ns.stream_id())
- .and_then(|s| s.topics.get(ns.topic_id()))
- .and_then(|t| t.partitions.get(ns.partition_id()))
- .map(|p| p.stats.clone())
- {
- size += stats.size_bytes_inconsistent();
- messages += stats.messages_count_inconsistent();
- }
- }
- (size, messages)
- };
-
- bytes.put_u32_le(topic_meta.id as u32);
- bytes.put_u64_le(topic_meta.created_at.into());
- bytes.put_u32_le(partition_ids.len() as u32);
- bytes.put_u64_le(topic_meta.message_expiry.into());
- bytes.put_u8(topic_meta.compression_algorithm.as_code());
- bytes.put_u64_le(topic_meta.max_topic_size.into());
- bytes.put_u8(topic_meta.replication_factor);
- bytes.put_u64_le(topic_size);
- bytes.put_u64_le(topic_messages);
- bytes.put_u8(topic_meta.name.len() as u8);
- bytes.put_slice(topic_meta.name.as_bytes());
- }
- }
-
- bytes.freeze()
- })
- }
-
- pub fn get_streams_from_metadata(&self) -> bytes::Bytes {
- self.metadata.with_metadata(|metadata| {
- let mut bytes = BytesMut::new();
-
- let mut stream_ids: Vec<_> = metadata.streams.iter().map(|(k, _)| k).collect();
- stream_ids.sort_unstable();
-
- for stream_id in stream_ids {
- let Some(stream_meta) = metadata.streams.get(stream_id) else {
- continue;
- };
-
- let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect();
- topic_ids.sort_unstable();
-
- let (total_size, total_messages) = {
- let mut size = 0u64;
- let mut messages = 0u64;
- for &topic_id in &topic_ids {
- if let Some(topic) = stream_meta.topics.get(topic_id) {
- for partition_id in 0..topic.partitions.len() {
- let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
- if let Some(stats) = metadata
- .streams
- .get(ns.stream_id())
- .and_then(|s| s.topics.get(ns.topic_id()))
- .and_then(|t| t.partitions.get(ns.partition_id()))
- .map(|p| p.stats.clone())
- {
- size += stats.size_bytes_inconsistent();
- messages += stats.messages_count_inconsistent();
- }
- }
- }
- }
- (size, messages)
- };
-
- bytes.put_u32_le(stream_meta.id as u32);
- bytes.put_u64_le(stream_meta.created_at.into());
- bytes.put_u32_le(topic_ids.len() as u32);
- bytes.put_u64_le(total_size);
- bytes.put_u64_le(total_messages);
- bytes.put_u8(stream_meta.name.len() as u8);
- bytes.put_slice(stream_meta.name.as_bytes());
- }
-
- bytes.freeze()
- })
- }
}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 8cdde46f3..dc87cda80 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -18,8 +18,8 @@
use crate::metadata::TopicMeta;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic};
use crate::streaming::topics::storage::{create_topic_file_hierarchy, delete_topic_directory};
-use bytes::{BufMut, BytesMut};
use iggy_common::sharding::IggyNamespace;
use iggy_common::{
CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize,
@@ -37,14 +37,14 @@ impl IggyShard {
#[allow(clippy::too_many_arguments)]
pub async fn create_topic(
&self,
- stream_id: &Identifier,
+ stream: ResolvedStream,
name: String,
message_expiry: IggyExpiry,
compression: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
) -> Result<usize, IggyError> {
- let stream_id = self.resolve_stream_id(stream_id)?;
+ let stream_id = stream.0;
let config = &self.config.system;
let message_expiry = config.resolve_message_expiry(message_expiry);
@@ -107,67 +107,40 @@ impl IggyShard {
#[allow(clippy::too_many_arguments)]
pub fn update_topic(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
name: String,
message_expiry: IggyExpiry,
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<(), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
- self.update_topic_base(
- stream,
- topic,
- name,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor.unwrap_or(1),
- )
- }
-
- #[allow(clippy::too_many_arguments)]
- fn update_topic_base(
- &self,
- stream: usize,
- topic: usize,
- name: String,
- message_expiry: IggyExpiry,
- compression_algorithm: CompressionAlgorithm,
- max_topic_size: MaxTopicSize,
- replication_factor: u8,
) -> Result<(), IggyError> {
self.writer().try_update_topic(
&self.metadata,
- stream,
- topic,
+ topic.stream_id,
+ topic.topic_id,
Arc::from(name.as_str()),
message_expiry,
compression_algorithm,
max_topic_size,
- replication_factor,
+ replication_factor.unwrap_or(1),
)
}
- pub async fn delete_topic(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- ) -> Result<DeletedTopicInfo, IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
+ pub async fn delete_topic(&self, topic: ResolvedTopic) -> Result<DeletedTopicInfo, IggyError> {
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
- let (partition_ids, messages_count, size_bytes, segments_count, parent_stats) =
+ let (partition_ids, topic_name, messages_count, size_bytes, segments_count, parent_stats) =
self.metadata.with_metadata(|m| {
let stream_meta = m.streams.get(stream).expect("Stream metadata must exist");
let topic_meta = stream_meta
.topics
- .get(topic)
+ .get(topic_id)
.expect("Topic metadata must exist");
let pids: Vec<usize> = (0..topic_meta.partitions.len()).collect();
(
pids,
+ topic_meta.name.to_string(),
topic_meta.stats.messages_count_inconsistent(),
topic_meta.stats.size_bytes_inconsistent(),
topic_meta.stats.segments_count_inconsistent(),
@@ -175,17 +148,31 @@ impl IggyShard {
)
});
- let topic_info = self.delete_topic_base(stream, topic);
+ {
+ let mut partitions = self.local_partitions.borrow_mut();
+ for &partition_id in &partition_ids {
+ let ns = IggyNamespace::new(stream, topic_id, partition_id);
+ partitions.remove(&ns);
+ }
+ }
+
+ self.writer().delete_topic(stream, topic_id);
+
+ let topic_info = DeletedTopicInfo {
+ id: topic_id,
+ name: topic_name,
+ stream_id: stream,
+ };
self.client_manager
- .delete_consumer_groups_for_topic(stream, topic_info.id);
+ .delete_consumer_groups_for_topic(stream, topic_id);
let namespaces_to_remove: Vec<_> = self
.shards_table
.iter()
.filter_map(|entry| {
let (ns, _) = entry.pair();
- if ns.stream_id() == stream && ns.topic_id() == topic_info.id {
+ if ns.stream_id() == stream && ns.topic_id() == topic_id {
Some(*ns)
} else {
None
@@ -197,7 +184,7 @@ impl IggyShard {
self.remove_shard_table_record(&ns);
}
- delete_topic_directory(stream, topic_info.id, &partition_ids, &self.config.system).await?;
+ delete_topic_directory(stream, topic_id, &partition_ids, &self.config.system).await?;
parent_stats.decrement_messages_count(messages_count);
parent_stats.decrement_size_bytes(size_bytes);
@@ -206,49 +193,17 @@ impl IggyShard {
Ok(topic_info)
}
- fn delete_topic_base(&self, stream: usize, topic: usize) -> DeletedTopicInfo {
- let (topic_name, partition_ids) = self.metadata.with_metadata(|m| {
- let stream_meta = m.streams.get(stream).expect("Stream metadata must exist");
- let topic_meta = stream_meta
- .topics
- .get(topic)
- .expect("Topic metadata must exist");
- let name = topic_meta.name.to_string();
- let pids: Vec<usize> = (0..topic_meta.partitions.len()).collect();
- (name, pids)
- });
+ pub async fn purge_topic(&self, topic: ResolvedTopic) -> Result<(), IggyError> {
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
- {
- let mut partitions = self.local_partitions.borrow_mut();
- for partition_id in partition_ids {
- let ns = IggyNamespace::new(stream, topic, partition_id);
- partitions.remove(&ns);
- }
- }
-
- self.writer().delete_topic(stream, topic);
-
- DeletedTopicInfo {
- id: topic,
- name: topic_name,
- stream_id: stream,
- }
- }
-
- pub async fn purge_topic(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- ) -> Result<(), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
-
- let partition_ids = self.metadata.get_partition_ids(stream, topic);
+ let partition_ids = self.metadata.get_partition_ids(stream, topic_id);
let mut all_consumer_paths = Vec::new();
let mut all_group_paths = Vec::new();
for partition_id in &partition_ids {
- let ns = IggyNamespace::new(stream, topic, *partition_id);
+ let ns = IggyNamespace::new(stream, topic_id, *partition_id);
if let Some(partition) = self.local_partitions.borrow().get(&ns) {
all_consumer_paths.extend(
partition
@@ -274,41 +229,30 @@ impl IggyShard {
self.delete_consumer_offset_from_disk(&path).await?;
}
- self.purge_topic_base(stream, topic).await
- }
-
- pub async fn purge_topic_bypass_auth(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- ) -> Result<(), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
- self.purge_topic_base(stream, topic).await
+ self.purge_topic_inner(topic).await
}
- pub(crate) async fn purge_topic_base(
- &self,
- stream: usize,
- topic: usize,
- ) -> Result<(), IggyError> {
- let partition_ids = self.metadata.get_partition_ids(stream, topic);
+ pub(crate) async fn purge_topic_inner(&self, topic: ResolvedTopic) -> Result<(), IggyError> {
+ let stream = topic.stream_id;
+ let topic_id = topic.topic_id;
+ let partition_ids = self.metadata.get_partition_ids(stream, topic_id);
for &partition_id in &partition_ids {
- let ns = IggyNamespace::new(stream, topic, partition_id);
+ let ns = IggyNamespace::new(stream, topic_id, partition_id);
let has_partition = self.local_partitions.borrow().contains(&ns);
if has_partition {
- self.delete_segments_base(stream, topic, partition_id, u32::MAX)
+ self.delete_segments(stream, topic_id, partition_id, u32::MAX)
.await?;
}
}
- if let Some(topic_stats) = self.metadata.get_topic_stats(stream, topic) {
+ if let Some(topic_stats) = self.metadata.get_topic_stats(stream, topic_id) {
topic_stats.zero_out_all();
}
for &partition_id in &partition_ids {
- let ns = IggyNamespace::new(stream, topic, partition_id);
+ let ns = IggyNamespace::new(stream, topic_id, partition_id);
if let Some(partition_stats) = self.metadata.get_partition_stats(&ns) {
partition_stats.zero_out_all();
}
@@ -316,139 +260,4 @@ impl IggyShard {
Ok(())
}
-
- pub fn get_topic_from_metadata(&self, stream_id: usize, topic_id: usize) -> bytes::Bytes {
- self.metadata.with_metadata(|metadata| {
- let Some(stream_meta) = metadata.streams.get(stream_id) else {
- return bytes::Bytes::new();
- };
- let Some(topic_meta) = stream_meta.topics.get(topic_id) else {
- return bytes::Bytes::new();
- };
-
- let mut partition_ids: Vec<_> = topic_meta
- .partitions
- .iter()
- .enumerate()
- .map(|(k, _)| k)
- .collect();
- partition_ids.sort_unstable();
-
- let (total_size, total_messages) = {
- let mut size = 0u64;
- let mut messages = 0u64;
- for &partition_id in &partition_ids {
- if let Some(stats) = metadata
- .streams
- .get(stream_id)
- .and_then(|s| s.topics.get(topic_id))
- .and_then(|t| t.partitions.get(partition_id))
- .map(|p| p.stats.clone())
- {
- size += stats.size_bytes_inconsistent();
- messages += stats.messages_count_inconsistent();
- }
- }
- (size, messages)
- };
-
- let mut bytes = BytesMut::new();
-
- bytes.put_u32_le(topic_meta.id as u32);
- bytes.put_u64_le(topic_meta.created_at.into());
- bytes.put_u32_le(partition_ids.len() as u32);
- bytes.put_u64_le(topic_meta.message_expiry.into());
- bytes.put_u8(topic_meta.compression_algorithm.as_code());
- bytes.put_u64_le(topic_meta.max_topic_size.into());
- bytes.put_u8(topic_meta.replication_factor);
- bytes.put_u64_le(total_size);
- bytes.put_u64_le(total_messages);
- bytes.put_u8(topic_meta.name.len() as u8);
- bytes.put_slice(topic_meta.name.as_bytes());
-
- for &partition_id in &partition_ids {
- let partition_meta = topic_meta.partitions.get(partition_id);
- let created_at = partition_meta
- .map(|m| m.created_at)
- .unwrap_or_else(IggyTimestamp::now);
-
- let (segments_count, size_bytes, messages_count, offset) = partition_meta
- .map(|p| {
- (
- p.stats.segments_count_inconsistent(),
- p.stats.size_bytes_inconsistent(),
- p.stats.messages_count_inconsistent(),
- p.stats.current_offset(),
- )
- })
- .unwrap_or((0, 0, 0, 0));
-
- bytes.put_u32_le(partition_id as u32);
- bytes.put_u64_le(created_at.into());
- bytes.put_u32_le(segments_count);
- bytes.put_u64_le(offset);
- bytes.put_u64_le(size_bytes);
- bytes.put_u64_le(messages_count);
- }
-
- bytes.freeze()
- })
- }
-
- pub fn get_topics_from_metadata(&self, stream_id: usize) -> bytes::Bytes {
- self.metadata.with_metadata(|metadata| {
- let mut bytes = BytesMut::new();
-
- let Some(stream_meta) = metadata.streams.get(stream_id) else {
- return bytes.freeze();
- };
-
- let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect();
- topic_ids.sort_unstable();
-
- for topic_id in topic_ids {
- let Some(topic_meta) = stream_meta.topics.get(topic_id) else {
- continue;
- };
-
- let mut partition_ids: Vec<_> = topic_meta
- .partitions
- .iter()
- .enumerate()
- .map(|(k, _)| k)
- .collect();
- partition_ids.sort_unstable();
-
- let (total_size, total_messages) = {
- let mut size = 0u64;
- let mut messages = 0u64;
- for &partition_id in &partition_ids {
- if let Some(stats) = topic_meta
- .partitions
- .get(partition_id)
- .map(|p| p.stats.clone())
- {
- size += stats.size_bytes_inconsistent();
- messages += stats.messages_count_inconsistent();
- }
- }
- (size, messages)
- };
-
- bytes.put_u32_le(topic_meta.id as u32);
- bytes.put_u64_le(topic_meta.created_at.into());
- bytes.put_u32_le(partition_ids.len() as u32);
- bytes.put_u64_le(topic_meta.message_expiry.into());
- bytes.put_u8(topic_meta.compression_algorithm.as_code());
- bytes.put_u64_le(topic_meta.max_topic_size.into());
- bytes.put_u8(topic_meta.replication_factor);
- bytes.put_u64_le(total_size);
- bytes.put_u64_le(total_messages);
- bytes.put_u8(topic_meta.name.len() as u8);
- bytes.put_slice(topic_meta.name.as_bytes());
- }
-
- bytes.freeze()
- })
- }
}
diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs
index 5ba69c2e9..ae2351700 100644
--- a/core/server/src/shard/system/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -117,10 +117,6 @@ impl IggyShard {
}
pub fn delete_user(&self, user_id: &Identifier) -> Result<User, IggyError> {
- self.delete_user_base(user_id)
- }
-
- fn delete_user_base(&self, user_id: &Identifier) -> Result<User, IggyError> {
let user = self.get_user(user_id).error(|e: &IggyError| {
format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}")
})?;
@@ -151,15 +147,6 @@ impl IggyShard {
user_id: &Identifier,
username: Option<String>,
status: Option<UserStatus>,
- ) -> Result<User, IggyError> {
- self.update_user_base(user_id, username, status)
- }
-
- fn update_user_base(
- &self,
- user_id: &Identifier,
- username: Option<String>,
- status: Option<UserStatus>,
) -> Result<User, IggyError> {
let user = self.get_user(user_id)?;
let numeric_user_id = user.id;
@@ -178,14 +165,6 @@ impl IggyShard {
&self,
user_id: &Identifier,
permissions: Option<Permissions>,
- ) -> Result<(), IggyError> {
- self.update_permissions_base(user_id, permissions)
- }
-
- fn update_permissions_base(
- &self,
- user_id: &Identifier,
- permissions: Option<Permissions>,
) -> Result<(), IggyError> {
let user: User = self.get_user(user_id).error(|e: &IggyError| {
format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}")
@@ -215,15 +194,6 @@ impl IggyShard {
user_id: &Identifier,
current_password: &str,
new_password: &str,
- ) -> Result<(), IggyError> {
- self.change_password_base(user_id, current_password, new_password)
- }
-
- fn change_password_base(
- &self,
- user_id: &Identifier,
- current_password: &str,
- new_password: &str,
) -> Result<(), IggyError> {
let user = self.get_user(user_id).error(|e: &IggyError| {
format!(
@@ -231,7 +201,6 @@ impl IggyShard {
)
})?;
- // Verify current password
if !crypto::verify_password(current_password, &user.password) {
error!(
"Invalid current password for user: {} with ID: {user_id}.",
@@ -325,11 +294,6 @@ impl IggyShard {
pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
let client_id = session.client_id;
- self.logout_user_base(client_id)?;
- Ok(())
- }
-
- fn logout_user_base(&self, client_id: u32) -> Result<(), IggyError> {
if client_id > 0 {
self.client_manager.clear_user_id(client_id)?;
}
diff --git a/core/server/src/shard/system/utils.rs b/core/server/src/shard/system/utils.rs
index 4919e3619..12ff45a42 100644
--- a/core/server/src/shard/system/utils.rs
+++ b/core/server/src/shard/system/utils.rs
@@ -15,115 +15,124 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{shard::IggyShard, streaming::polling_consumer::PollingConsumer};
-use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError};
+use crate::{
+ metadata::InnerMetadata,
+ shard::{
+ IggyShard,
+ transmission::message::{
+ ResolvedConsumerGroup, ResolvedPartition, ResolvedStream, ResolvedTopic,
+ },
+ },
+ streaming::polling_consumer::PollingConsumer,
+};
+use iggy_common::{Consumer, ConsumerKind, IdKind, Identifier, IggyError};
+use std::sync::Arc;
impl IggyShard {
- /// Resolves stream identifier to numeric ID, returning error if not found.
- pub fn resolve_stream_id(&self, stream_id: &Identifier) -> Result<usize, IggyError> {
+ /// Resolves stream identifier to typed `ResolvedStream`.
+ pub fn resolve_stream(&self, stream_id: &Identifier) -> Result<ResolvedStream, IggyError> {
self.metadata
- .get_stream_id(stream_id)
- .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))
+ .with_metadata(|m| resolve_stream_inner(m, stream_id))
}
- /// Resolves topic identifier to (stream_id, topic_id), returning error if not found.
- pub fn resolve_topic_id(
+ /// Resolves topic from identifiers. Returns StreamIdNotFound if stream doesn't exist,
+ /// TopicIdNotFound if topic doesn't exist.
+ pub fn resolve_topic(
&self,
stream_id: &Identifier,
topic_id: &Identifier,
- ) -> Result<(usize, usize), IggyError> {
- let stream = self.resolve_stream_id(stream_id)?;
- let topic = self
- .metadata
- .get_topic_id(stream, topic_id)
- .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
- Ok((stream, topic))
+ ) -> Result<ResolvedTopic, IggyError> {
+ self.metadata.with_metadata(|m| {
+ let stream = resolve_stream_inner(m, stream_id)?;
+ let id = resolve_topic_id_inner(m, stream.0, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+ Ok(ResolvedTopic {
+ stream_id: stream.0,
+ topic_id: id,
+ })
+ })
}
- /// Resolves partition identifier to (stream_id, topic_id, partition_id), returning error if not found.
- pub fn resolve_partition_id(
+ /// Resolves partition from identifiers. Returns appropriate error at each level.
+ pub fn resolve_partition(
&self,
stream_id: &Identifier,
topic_id: &Identifier,
partition_id: usize,
- ) -> Result<(usize, usize, usize), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
- if !self.metadata.partition_exists(stream, topic, partition_id) {
- return Err(IggyError::PartitionNotFound(
+ ) -> Result<ResolvedPartition, IggyError> {
+ self.metadata.with_metadata(|m| {
+ let stream = resolve_stream_inner(m, stream_id)?;
+ let tid = resolve_topic_id_inner(m, stream.0, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
+
+ let exists = m
+ .streams
+ .get(stream.0)
+ .and_then(|s| s.topics.get(tid))
+ .and_then(|t| t.partitions.get(partition_id))
+ .is_some();
+
+ if !exists {
+ return Err(IggyError::PartitionNotFound(
+ partition_id,
+ topic_id.clone(),
+ stream_id.clone(),
+ ));
+ }
+
+ Ok(ResolvedPartition {
+ stream_id: stream.0,
+ topic_id: tid,
partition_id,
- topic_id.clone(),
- stream_id.clone(),
- ));
- }
- Ok((stream, topic, partition_id))
+ })
+ })
}
- /// Resolves consumer group identifier to (stream_id, topic_id, group_id), returning error if not found.
- pub fn resolve_consumer_group_id(
+ /// Resolves consumer group from identifiers. Returns appropriate error at each level.
+ pub fn resolve_consumer_group(
&self,
stream_id: &Identifier,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> Result<(usize, usize, usize), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
- let group = self
- .metadata
- .get_consumer_group_id(stream, topic, group_id)
- .ok_or_else(|| {
- IggyError::ConsumerGroupIdNotFound(group_id.clone(), topic_id.clone())
- })?;
- Ok((stream, topic, group))
- }
+ ) -> Result<ResolvedConsumerGroup, IggyError> {
+ self.metadata.with_metadata(|m| {
+ let stream = resolve_stream_inner(m, stream_id)?;
+ let tid = resolve_topic_id_inner(m, stream.0, topic_id)
+ .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?;
- pub fn ensure_topic_exists(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- ) -> Result<(), IggyError> {
- self.resolve_topic_id(stream_id, topic_id)?;
- Ok(())
- }
+ let gid =
+ resolve_consumer_group_id_inner(m, stream.0, tid, group_id).ok_or_else(|| {
+ IggyError::ConsumerGroupIdNotFound(group_id.clone(), topic_id.clone())
+ })?;
- pub fn ensure_consumer_group_exists(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- group_id: &Identifier,
- ) -> Result<(), IggyError> {
- self.resolve_consumer_group_id(stream_id, topic_id, group_id)?;
- Ok(())
+ Ok(ResolvedConsumerGroup {
+ stream_id: stream.0,
+ topic_id: tid,
+ group_id: gid,
+ })
+ })
}
- pub fn ensure_partitions_exist(
+ /// Validates that partitions_count does not exceed actual partition count.
+ pub fn validate_partitions_count(
&self,
- stream_id: &Identifier,
- topic_id: &Identifier,
+ topic: ResolvedTopic,
partitions_count: u32,
) -> Result<(), IggyError> {
- let (stream, topic) = self.resolve_topic_id(stream_id, topic_id)?;
- let actual_partitions_count = self.metadata.partitions_count(stream, topic);
-
- if partitions_count > actual_partitions_count as u32 {
+ let actual = self
+ .metadata
+ .partitions_count(topic.stream_id, topic.topic_id);
+ if partitions_count > actual as u32 {
return Err(IggyError::InvalidPartitionsCount);
}
-
- Ok(())
- }
-
- pub fn ensure_partition_exists(
- &self,
- stream_id: &Identifier,
- topic_id: &Identifier,
- partition_id: usize,
- ) -> Result<(), IggyError> {
- self.resolve_partition_id(stream_id, topic_id, partition_id)?;
Ok(())
... 634 lines suppressed ...