You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/07/08 14:10:56 UTC
(iggy) 01/01: feat(io_uring): fix delete stream/topic events
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch fix_delete_stream_and_topic_events
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 2fbbfa586b2afc6f1011849979cc692183ac425f
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jul 8 16:10:46 2025 +0200
feat(io_uring): fix delete stream/topic events
---
.../handlers/streams/delete_stream_handler.rs | 44 +++++++++++++++++++++-
.../binary/handlers/topics/delete_topic_handler.rs | 41 +++++++++++++++++---
core/server/src/shard/mod.rs | 29 +++++++++++---
core/server/src/shard/system/streams.rs | 32 ++++++++++------
core/server/src/shard/system/topics.rs | 43 +++++++++++++--------
core/server/src/streaming/streams/persistence.rs | 6 ---
core/server/src/streaming/topics/persistence.rs | 10 -----
7 files changed, 147 insertions(+), 58 deletions(-)
diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index a0e52a43..36cb3454 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -20,13 +20,18 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
+use crate::streaming::partitions::partition;
use crate::streaming::session::Session;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_stream::DeleteStream;
+use iggy_common::locking::IggySharedMutFn;
use std::rc::Rc;
+use std::time::Duration;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteStream {
@@ -45,12 +50,47 @@ impl ServerCommandHandler for DeleteStream {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
- shard
+ let stream = shard
.delete_stream(session, &self.stream_id)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete stream with ID: {stream_id}, session: {session}")
})?;
+ // First we have to go over all of the partitions inside of topics
+ // and make sure that they are deleted.
+ for topic in stream.get_topics() {
+ let partitions = topic.get_partitions();
+ let mut namespaces = Vec::new();
+ for partition in partitions {
+ let partition_id = partition.read().await.partition_id;
+ let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
+ namespaces.push(namespace);
+ }
+ let records = shard.remove_shard_table_records(&namespaces);
+ for (ns, shard_info) in records {
+ if shard_info.id() == shard.id {
+ let partition = topic.get_partition(ns.partition_id).unwrap();
+ let mut partition = partition.write().await;
+ let partition_id = partition.partition_id;
+ partition.delete().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to delete partition in stream: {self}, partition: {partition_id}")
+ })?;
+ }
+ }
+ let event = ShardEvent::DeletedShardTableRecords { namespaces };
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
+ //TODO: Once event response is implemented, we could get rid of this.
+ compio::time::sleep(Duration::from_millis(50)).await;
+ topic.delete().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
+ })?;
+ }
+ if stream.delete().await.is_err() {
+ return Err(IggyError::CannotDeleteStream(stream.stream_id));
+ }
+ let event = ShardEvent::DeletedStream {
+ stream_id: self.stream_id.clone(),
+ };
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
shard
.state
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index a8978511..431db62f 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -20,13 +20,17 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_topic::DeleteTopic;
+use iggy_common::locking::IggySharedMutFn;
use std::rc::Rc;
+use std::time::Duration;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteTopic {
@@ -43,22 +47,47 @@ impl ServerCommandHandler for DeleteTopic {
shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
-
- shard
+ let topic = shard
.delete_topic(session, &self.stream_id, &self.topic_id)
.await
.with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
+ "{COMPONENT} (error: {error}) - failed to delete topic with ID: {} in stream with ID: {}, session: {session}",
+ &self.topic_id, &self.stream_id
))?;
+ let numeric_topic_id = topic.topic_id;
+ let numeric_stream_id = topic.stream_id;
+ let partitions = topic.get_partitions();
+ let mut namespaces = Vec::new();
+ for partition in partitions {
+ let partition_id = partition.read().await.partition_id;
+ let namespace = IggyNamespace::new(numeric_stream_id, topic.topic_id, partition_id);
+ namespaces.push(namespace);
+ }
+ let records = shard.remove_shard_table_records(&namespaces);
+ for (ns, shard_info) in records {
+ if shard_info.id() == shard.id {
+ let partition = topic.get_partition(ns.partition_id).unwrap();
+ let mut partition = partition.write().await;
+ let partition_id = partition.partition_id;
+ partition.delete().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to delete partition in stream: {self}, partition: {partition_id}")
+ })?;
+ }
+ }
+ let event = ShardEvent::DeletedShardTableRecords { namespaces };
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
+ //TODO: Once event response is implemented, we could get rid of this.
+ compio::time::sleep(Duration::from_millis(50)).await;
+ topic.delete().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
+ })?;
shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
.await
.with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - failed to apply delete topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
+ "{COMPONENT} (error: {error}) - failed to apply delete topic with ID: {numeric_topic_id} in stream with ID: {numeric_stream_id}, session: {session}",
))?;
sender.send_empty_ok_response().await?;
Ok(())
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index e187bd04..b46073c2 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -607,6 +607,7 @@ impl IggyShard {
let mut segments_count = 0;
let mut messages_count = 0;
+ let partitions_count = partitions.len();
for partition in &partitions {
let partition = partition.read().await;
let partition_messages_count = partition.get_messages_count();
@@ -626,7 +627,7 @@ impl IggyShard {
})?;
topic.reassign_consumer_groups();
if partitions.len() > 0 {
- self.metrics.decrement_partitions(0);
+ self.metrics.decrement_partitions(partitions_count as u32);
self.metrics.decrement_segments(segments_count);
self.metrics.decrement_messages(messages_count);
}
@@ -655,8 +656,15 @@ impl IggyShard {
}
Ok(())
}
-
- ShardEvent::DeletedStream { stream_id: _ } => todo!(),
+ ShardEvent::DeletedStream { stream_id } => {
+ let shard_id = self.id;
+ self.delete_stream_bypass_auth(stream_id).with_error_context(|err| {
+ format!(
+ "{COMPONENT} (error: {err}) - failed to delete, when handling event on shard: {shard_id} stream with ID: {stream_id}",
+ )
+ })?;
+ Ok(())
+ }
ShardEvent::UpdatedStream {
stream_id: _,
name: _,
@@ -687,9 +695,18 @@ impl IggyShard {
topic_id: _,
} => todo!(),
ShardEvent::DeletedTopic {
- stream_id: _,
- topic_id: _,
- } => todo!(),
+ stream_id,
+ topic_id,
+ } => {
+ self.delete_topic_bypass_auth(stream_id, topic_id)
+ .await
+ .with_error_context(|err| {
+ format!(
+ "{COMPONENT} (error: {err}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"
+ )
+ })?;
+ Ok(())
+ }
ShardEvent::CreatedUser {
username: _,
password: _,
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index bc8fc2bb..e884bd4a 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -319,16 +319,23 @@ impl IggyShard {
Ok(())
}
- pub async fn delete_stream(
- &self,
- session: &Session,
- id: &Identifier,
- ) -> Result<u32, IggyError> {
+ pub fn delete_stream_bypass_auth(&self, id: &Identifier) -> Result<Stream, IggyError> {
+ let stream = self.get_stream(id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let stream_name = stream.name.clone();
+ let stream = self.delete_stream_base(stream_id, stream_name)?;
+ Ok(stream)
+ }
+
+ pub fn delete_stream(&self, session: &Session, id: &Identifier) -> Result<Stream, IggyError> {
self.ensure_authenticated(session)?;
let stream = self.get_stream(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
})?;
let stream_id = stream.stream_id;
+ let stream_name = stream.name.clone();
self.permissioner
.borrow()
.delete_stream(session.get_user_id(), stream_id)
@@ -339,10 +346,13 @@ impl IggyShard {
stream.stream_id,
)
})?;
- let stream_name = stream.name.clone();
- if stream.delete().await.is_err() {
- return Err(IggyError::CannotDeleteStream(stream_id));
- }
+ let stream = self.delete_stream_base(stream_id, stream_name)?;
+ Ok(stream)
+ }
+
+ fn delete_stream_base(&self, stream_id: u32, stream_name: String) -> Result<Stream, IggyError> {
+ let stream = self.streams.borrow_mut().remove(&stream_id).unwrap();
+ self.streams_ids.borrow_mut().remove(&stream_name);
self.metrics.decrement_streams(1);
self.metrics.decrement_topics(stream.get_topics_count());
@@ -350,8 +360,6 @@ impl IggyShard {
.decrement_partitions(stream.get_partitions_count());
self.metrics.decrement_messages(stream.get_messages_count());
self.metrics.decrement_segments(stream.get_segments_count());
- self.streams.borrow_mut().remove(&stream_id);
- self.streams_ids.borrow_mut().remove(&stream_name);
let current_stream_id = CURRENT_STREAM_ID.load(Ordering::SeqCst);
if current_stream_id > stream_id {
CURRENT_STREAM_ID.store(stream_id, Ordering::SeqCst);
@@ -360,7 +368,7 @@ impl IggyShard {
self.client_manager
.borrow_mut()
.delete_consumer_groups_for_stream(stream_id);
- Ok(stream_id)
+ Ok(stream)
}
pub async fn purge_stream(
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 31562504..e2cce4d7 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -301,14 +301,22 @@ impl IggyShard {
Ok(())
}
+ pub async fn delete_topic_bypass_auth(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<Topic, IggyError> {
+ let topic = self.delete_topic_base(stream_id, topic_id).await?;
+ Ok(topic)
+ }
+
pub async fn delete_topic(
&self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
- ) -> Result<(), IggyError> {
+ ) -> Result<Topic, IggyError> {
self.ensure_authenticated(session)?;
- let stream_id_value;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
@@ -328,24 +336,27 @@ impl IggyShard {
session.get_user_id(),
)
})?;
- stream_id_value = topic.stream_id;
}
- let topic = self
- .get_stream_mut(stream_id)?
- .delete_topic(topic_id)
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
- let stream = self.get_stream(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
- })?;
- topic
- .delete()
+ let topic = self.delete_topic_base(stream_id, topic_id)
.await
.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to delete topic: {topic}")
- })
- .map_err(|_| IggyError::CannotDeleteTopic(topic.topic_id, stream.stream_id))?;
+ format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}")
+ })?;
+ Ok(topic)
+ }
+ async fn delete_topic_base(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ ) -> Result<Topic, IggyError> {
+ let mut stream = self.get_stream_mut(stream_id)?;
+ let stream_id_value = stream.stream_id;
+ let topic = stream
+ .delete_topic(topic_id)
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
+ drop(stream);
self.metrics.decrement_topics(1);
self.metrics
.decrement_partitions(topic.get_partitions_count());
@@ -355,7 +366,7 @@ impl IggyShard {
self.client_manager
.borrow_mut()
.delete_consumer_groups_for_topic(stream_id_value, topic.topic_id);
- Ok(())
+ Ok(topic)
}
pub async fn purge_topic(
diff --git a/core/server/src/streaming/streams/persistence.rs b/core/server/src/streaming/streams/persistence.rs
index 3f5cae9e..c24cdcf8 100644
--- a/core/server/src/streaming/streams/persistence.rs
+++ b/core/server/src/streaming/streams/persistence.rs
@@ -43,12 +43,6 @@ impl Stream {
}
pub async fn delete(&self) -> Result<(), IggyError> {
- for topic in self.get_topics() {
- topic.delete().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
- })?;
- }
-
self.storage
.stream
.delete(self)
diff --git a/core/server/src/streaming/topics/persistence.rs b/core/server/src/streaming/topics/persistence.rs
index f75e36ec..1fcbd182 100644
--- a/core/server/src/streaming/topics/persistence.rs
+++ b/core/server/src/streaming/topics/persistence.rs
@@ -35,16 +35,6 @@ impl Topic {
}
pub async fn delete(&self) -> Result<(), IggyError> {
- for partition in self.get_partitions() {
- let mut partition = partition.write().await;
- partition.delete().await.with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to delete partition with ID: {} in topic with ID: {}",
- partition.partition_id, self.topic_id
- )
- })?;
- }
-
self.storage.topic.delete(self).await
}