You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/07/08 14:10:56 UTC

(iggy) 01/01: feat(io_uring): fix delete stream/topic events

This is an automated email from the ASF dual-hosted git repository.

gkoszyk pushed a commit to branch fix_delete_stream_and_topic_events
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit 2fbbfa586b2afc6f1011849979cc692183ac425f
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jul 8 16:10:46 2025 +0200

    feat(io_uring): fix delete stream/topic events
---
 .../handlers/streams/delete_stream_handler.rs      | 44 +++++++++++++++++++++-
 .../binary/handlers/topics/delete_topic_handler.rs | 41 +++++++++++++++++---
 core/server/src/shard/mod.rs                       | 29 +++++++++++---
 core/server/src/shard/system/streams.rs            | 32 ++++++++++------
 core/server/src/shard/system/topics.rs             | 43 +++++++++++++--------
 core/server/src/streaming/streams/persistence.rs   |  6 ---
 core/server/src/streaming/topics/persistence.rs    | 10 -----
 7 files changed, 147 insertions(+), 58 deletions(-)

diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index a0e52a43..36cb3454 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -20,13 +20,18 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
 use crate::binary::handlers::utils::receive_and_validate;
 use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
 use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::event::ShardEvent;
 use crate::state::command::EntryCommand;
+use crate::streaming::partitions::partition;
 use crate::streaming::session::Session;
 use anyhow::Result;
 use error_set::ErrContext;
 use iggy_common::IggyError;
 use iggy_common::delete_stream::DeleteStream;
+use iggy_common::locking::IggySharedMutFn;
 use std::rc::Rc;
+use std::time::Duration;
 use tracing::{debug, instrument};
 
 impl ServerCommandHandler for DeleteStream {
@@ -45,12 +50,47 @@ impl ServerCommandHandler for DeleteStream {
         debug!("session: {session}, command: {self}");
         let stream_id = self.stream_id.clone();
 
-        shard
+        let stream = shard
                 .delete_stream(session, &self.stream_id)
-                .await
                 .with_error_context(|error| {
                     format!("{COMPONENT} (error: {error}) - failed to delete stream with ID: {stream_id}, session: {session}")
                 })?;
+        // First we have to go over all of the partitions inside of topics
+        // and make sure that they are deleted.
+        for topic in stream.get_topics() {
+            let partitions = topic.get_partitions();
+            let mut namespaces = Vec::new();
+            for partition in partitions {
+                let partition_id = partition.read().await.partition_id;
+                let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
+                namespaces.push(namespace);
+            }
+            let records = shard.remove_shard_table_records(&namespaces);
+            for (ns, shard_info) in records {
+                if shard_info.id() == shard.id {
+                    let partition = topic.get_partition(ns.partition_id).unwrap();
+                    let mut partition = partition.write().await;
+                    let partition_id = partition.partition_id;
+                    partition.delete().await.with_error_context(|error| {
+                        format!("{COMPONENT} (error: {error}) - failed to delete partition in stream: {self}, partition: {partition_id}")
+                    })?;
+                }
+            }
+            let event = ShardEvent::DeletedShardTableRecords { namespaces };
+            let _responses = shard.broadcast_event_to_all_shards(event.into());
+            //TODO: Once event response is implemented, we could get rid of this.
+            compio::time::sleep(Duration::from_millis(50)).await;
+            topic.delete().await.with_error_context(|error| {
+                format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
+            })?;
+        }
+        if stream.delete().await.is_err() {
+            return Err(IggyError::CannotDeleteStream(stream.stream_id));
+        }
+        let event = ShardEvent::DeletedStream {
+            stream_id: self.stream_id.clone(),
+        };
+        let _responses = shard.broadcast_event_to_all_shards(event.into());
 
         shard
             .state
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index a8978511..431db62f 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -20,13 +20,17 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
 use crate::binary::handlers::utils::receive_and_validate;
 use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
 use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::event::ShardEvent;
 use crate::state::command::EntryCommand;
 use crate::streaming::session::Session;
 use anyhow::Result;
 use error_set::ErrContext;
 use iggy_common::IggyError;
 use iggy_common::delete_topic::DeleteTopic;
+use iggy_common::locking::IggySharedMutFn;
 use std::rc::Rc;
+use std::time::Duration;
 use tracing::{debug, instrument};
 
 impl ServerCommandHandler for DeleteTopic {
@@ -43,22 +47,47 @@ impl ServerCommandHandler for DeleteTopic {
         shard: &Rc<IggyShard>,
     ) -> Result<(), IggyError> {
         debug!("session: {session}, command: {self}");
-        let stream_id = self.stream_id.clone();
-        let topic_id = self.topic_id.clone();
-
-        shard
+        let topic = shard
                 .delete_topic(session, &self.stream_id, &self.topic_id)
                 .await
                 .with_error_context(|error| format!(
-                    "{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
+                    "{COMPONENT} (error: {error}) - failed to delete topic with ID: {} in stream with ID: {}, session: {session}",
+                    &self.topic_id, &self.stream_id
                 ))?;
+        let numeric_topic_id = topic.topic_id;
+        let numeric_stream_id = topic.stream_id;
+        let partitions = topic.get_partitions();
+        let mut namespaces = Vec::new();
+        for partition in partitions {
+            let partition_id = partition.read().await.partition_id;
+            let namespace = IggyNamespace::new(numeric_stream_id, topic.topic_id, partition_id);
+            namespaces.push(namespace);
+        }
+        let records = shard.remove_shard_table_records(&namespaces);
+        for (ns, shard_info) in records {
+            if shard_info.id() == shard.id {
+                let partition = topic.get_partition(ns.partition_id).unwrap();
+                let mut partition = partition.write().await;
+                let partition_id = partition.partition_id;
+                partition.delete().await.with_error_context(|error| {
+                        format!("{COMPONENT} (error: {error}) - failed to delete partition in stream: {self}, partition: {partition_id}")
+                    })?;
+            }
+        }
+        let event = ShardEvent::DeletedShardTableRecords { namespaces };
+        let _responses = shard.broadcast_event_to_all_shards(event.into());
+        //TODO: Once event response is implemented, we could get rid of this.
+        compio::time::sleep(Duration::from_millis(50)).await;
+        topic.delete().await.with_error_context(|error| {
+            format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
+        })?;
 
         shard
             .state
             .apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
             .await
             .with_error_context(|error| format!(
-                "{COMPONENT} (error: {error}) - failed to apply delete topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
+                "{COMPONENT} (error: {error}) - failed to apply delete topic with ID: {numeric_topic_id} in stream with ID: {numeric_stream_id}, session: {session}",
             ))?;
         sender.send_empty_ok_response().await?;
         Ok(())
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index e187bd04..b46073c2 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -607,6 +607,7 @@ impl IggyShard {
 
                 let mut segments_count = 0;
                 let mut messages_count = 0;
+                let partitions_count = partitions.len();
                 for partition in &partitions {
                     let partition = partition.read().await;
                     let partition_messages_count = partition.get_messages_count();
@@ -626,7 +627,7 @@ impl IggyShard {
                 })?;
                 topic.reassign_consumer_groups();
                 if partitions.len() > 0 {
-                    self.metrics.decrement_partitions(0);
+                    self.metrics.decrement_partitions(partitions_count as u32);
                     self.metrics.decrement_segments(segments_count);
                     self.metrics.decrement_messages(messages_count);
                 }
@@ -655,8 +656,15 @@ impl IggyShard {
                 }
                 Ok(())
             }
-
-            ShardEvent::DeletedStream { stream_id: _ } => todo!(),
+            ShardEvent::DeletedStream { stream_id } => {
+                let shard_id = self.id;
+                self.delete_stream_bypass_auth(stream_id).with_error_context(|err| {
+                    format!(
+                        "{COMPONENT} (error: {err}) - failed to delete, when handling event on shard: {shard_id} stream with ID: {stream_id}",
+                    )
+                })?;
+                Ok(())
+            }
             ShardEvent::UpdatedStream {
                 stream_id: _,
                 name: _,
@@ -687,9 +695,18 @@ impl IggyShard {
                 topic_id: _,
             } => todo!(),
             ShardEvent::DeletedTopic {
-                stream_id: _,
-                topic_id: _,
-            } => todo!(),
+                stream_id,
+                topic_id,
+            } => {
+                self.delete_topic_bypass_auth(stream_id, topic_id)
+                    .await
+                    .with_error_context(|err| {
+                        format!(
+                            "{COMPONENT} (error: {err}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"
+                        )
+                    })?;
+                Ok(())
+            }
             ShardEvent::CreatedUser {
                 username: _,
                 password: _,
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index bc8fc2bb..e884bd4a 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -319,16 +319,23 @@ impl IggyShard {
         Ok(())
     }
 
-    pub async fn delete_stream(
-        &self,
-        session: &Session,
-        id: &Identifier,
-    ) -> Result<u32, IggyError> {
+    pub fn delete_stream_bypass_auth(&self, id: &Identifier) -> Result<Stream, IggyError> {
+        let stream = self.get_stream(id).with_error_context(|error| {
+            format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
+        })?;
+        let stream_id = stream.stream_id;
+        let stream_name = stream.name.clone();
+        let stream = self.delete_stream_base(stream_id, stream_name)?;
+        Ok(stream)
+    }
+
+    pub fn delete_stream(&self, session: &Session, id: &Identifier) -> Result<Stream, IggyError> {
         self.ensure_authenticated(session)?;
         let stream = self.get_stream(id).with_error_context(|error| {
             format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
         })?;
         let stream_id = stream.stream_id;
+        let stream_name = stream.name.clone();
         self.permissioner
             .borrow()
             .delete_stream(session.get_user_id(), stream_id)
@@ -339,10 +346,13 @@ impl IggyShard {
                     stream.stream_id,
                 )
             })?;
-        let stream_name = stream.name.clone();
-        if stream.delete().await.is_err() {
-            return Err(IggyError::CannotDeleteStream(stream_id));
-        }
+        let stream = self.delete_stream_base(stream_id, stream_name)?;
+        Ok(stream)
+    }
+
+    fn delete_stream_base(&self, stream_id: u32, stream_name: String) -> Result<Stream, IggyError> {
+        let stream = self.streams.borrow_mut().remove(&stream_id).unwrap();
+        self.streams_ids.borrow_mut().remove(&stream_name);
 
         self.metrics.decrement_streams(1);
         self.metrics.decrement_topics(stream.get_topics_count());
@@ -350,8 +360,6 @@ impl IggyShard {
             .decrement_partitions(stream.get_partitions_count());
         self.metrics.decrement_messages(stream.get_messages_count());
         self.metrics.decrement_segments(stream.get_segments_count());
-        self.streams.borrow_mut().remove(&stream_id);
-        self.streams_ids.borrow_mut().remove(&stream_name);
         let current_stream_id = CURRENT_STREAM_ID.load(Ordering::SeqCst);
         if current_stream_id > stream_id {
             CURRENT_STREAM_ID.store(stream_id, Ordering::SeqCst);
@@ -360,7 +368,7 @@ impl IggyShard {
         self.client_manager
             .borrow_mut()
             .delete_consumer_groups_for_stream(stream_id);
-        Ok(stream_id)
+        Ok(stream)
     }
 
     pub async fn purge_stream(
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 31562504..e2cce4d7 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -301,14 +301,22 @@ impl IggyShard {
         Ok(())
     }
 
+    pub async fn delete_topic_bypass_auth(
+        &self,
+        stream_id: &Identifier,
+        topic_id: &Identifier,
+    ) -> Result<Topic, IggyError> {
+        let topic = self.delete_topic_base(stream_id, topic_id).await?;
+        Ok(topic)
+    }
+
     pub async fn delete_topic(
         &self,
         session: &Session,
         stream_id: &Identifier,
         topic_id: &Identifier,
-    ) -> Result<(), IggyError> {
+    ) -> Result<Topic, IggyError> {
         self.ensure_authenticated(session)?;
-        let stream_id_value;
         {
             let stream = self.get_stream(stream_id).with_error_context(|error| {
                 format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
@@ -328,24 +336,27 @@ impl IggyShard {
                     session.get_user_id(),
                 )
             })?;
-            stream_id_value = topic.stream_id;
         }
 
-        let topic = self
-            .get_stream_mut(stream_id)?
-            .delete_topic(topic_id)
-            .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
-        let stream = self.get_stream(stream_id).with_error_context(|error| {
-            format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
-        })?;
-        topic
-            .delete()
+        let topic = self.delete_topic_base(stream_id, topic_id)
             .await
             .with_error_context(|error| {
-                format!("{COMPONENT} (error: {error}) - failed to delete topic: {topic}")
-            })
-            .map_err(|_| IggyError::CannotDeleteTopic(topic.topic_id, stream.stream_id))?;
+                format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}")
+            })?;
+        Ok(topic)
+    }
 
+    async fn delete_topic_base(
+        &self,
+        stream_id: &Identifier,
+        topic_id: &Identifier,
+    ) -> Result<Topic, IggyError> {
+        let mut stream = self.get_stream_mut(stream_id)?;
+        let stream_id_value = stream.stream_id;
+        let topic = stream
+            .delete_topic(topic_id)
+            .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
+        drop(stream);
         self.metrics.decrement_topics(1);
         self.metrics
             .decrement_partitions(topic.get_partitions_count());
@@ -355,7 +366,7 @@ impl IggyShard {
         self.client_manager
             .borrow_mut()
             .delete_consumer_groups_for_topic(stream_id_value, topic.topic_id);
-        Ok(())
+        Ok(topic)
     }
 
     pub async fn purge_topic(
diff --git a/core/server/src/streaming/streams/persistence.rs b/core/server/src/streaming/streams/persistence.rs
index 3f5cae9e..c24cdcf8 100644
--- a/core/server/src/streaming/streams/persistence.rs
+++ b/core/server/src/streaming/streams/persistence.rs
@@ -43,12 +43,6 @@ impl Stream {
     }
 
     pub async fn delete(&self) -> Result<(), IggyError> {
-        for topic in self.get_topics() {
-            topic.delete().await.with_error_context(|error| {
-                format!("{COMPONENT} (error: {error}) - failed to delete topic in stream: {self}")
-            })?;
-        }
-
         self.storage
             .stream
             .delete(self)
diff --git a/core/server/src/streaming/topics/persistence.rs b/core/server/src/streaming/topics/persistence.rs
index f75e36ec..1fcbd182 100644
--- a/core/server/src/streaming/topics/persistence.rs
+++ b/core/server/src/streaming/topics/persistence.rs
@@ -35,16 +35,6 @@ impl Topic {
     }
 
     pub async fn delete(&self) -> Result<(), IggyError> {
-        for partition in self.get_partitions() {
-            let mut partition = partition.write().await;
-            partition.delete().await.with_error_context(|error| {
-                format!(
-                    "{COMPONENT} (error: {error}) - failed to delete partition with ID: {} in topic with ID: {}",
-                    partition.partition_id, self.topic_id
-                )
-            })?;
-        }
-
         self.storage.topic.delete(self).await
     }