You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/07/14 14:15:09 UTC

(iggy) branch fix_shard_integration_tests created (now 5af88dd3)

This is an automated email from the ASF dual-hosted git repository.

gkoszyk pushed a change to branch fix_shard_integration_tests
in repository https://gitbox.apache.org/repos/asf/iggy.git


      at 5af88dd3 feat(io_uring): fix shard integration tests

This branch includes the following new commits:

     new 5af88dd3 feat(io_uring): fix shard integration tests

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(iggy) 01/01: feat(io_uring): fix shard integration tests

Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gkoszyk pushed a commit to branch fix_shard_integration_tests
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit 5af88dd37195a7303184b540cfc8b00c8c9d3fc7
Author: numminex <g....@gmail.com>
AuthorDate: Mon Jul 14 16:14:57 2025 +0200

    feat(io_uring): fix shard integration tests
---
 core/integration/tests/streaming/shard.rs | 133 ++++++++++++++++++------------
 core/server/src/io/fs_utils.rs            |  23 ++++--
 core/server/src/shard/mod.rs              |   3 +-
 core/server/src/shard/system/streams.rs   |  22 +++--
 4 files changed, 112 insertions(+), 69 deletions(-)

diff --git a/core/integration/tests/streaming/shard.rs b/core/integration/tests/streaming/shard.rs
index 454a5841..3a32fc5a 100644
--- a/core/integration/tests/streaming/shard.rs
+++ b/core/integration/tests/streaming/shard.rs
@@ -18,106 +18,133 @@
 
 use crate::streaming::common::test_setup::TestSetup;
 use iggy::prelude::Identifier;
-use server::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
+use iggy_common::locking::IggyRwLockFn;
+use server::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig, ServerConfig};
+use server::configs::system::SystemConfig;
+use server::shard::IggyShard;
 use server::streaming::session::Session;
 use std::net::{Ipv4Addr, SocketAddr};
-use tokio::fs;
+use std::rc::Rc;
+use compio::fs;
 
 //TODO: Fix me use shard instead of system
-/*
-#[tokio::test]
+#[compio::test]
 async fn should_initialize_system_and_base_directories() {
     let setup = TestSetup::init().await;
-    let mut system = System::new(
-        setup.config.clone(),
-        DataMaintenanceConfig::default(),
-        PersonalAccessTokenConfig::default(),
-    );
-
-    system.init().await.unwrap();
-
-    let mut dir_entries = fs::read_dir(&setup.config.path).await.unwrap();
+    let server_config = ServerConfig {
+        system:  setup.config.clone(),
+        ..Default::default()
+    };
+    let shard = IggyShard::default_from_config(server_config);
+    shard.init().await.unwrap();
+
+    let mut dir_entries = server::io::fs_utils::walk_dir(&setup.config.path).await.unwrap();
     let mut names = Vec::new();
-    while let Some(entry) = dir_entries.next_entry().await.unwrap() {
-        let metadata = entry.metadata().await.unwrap();
-        assert!(metadata.is_dir());
-        names.push(entry.file_name().into_string().unwrap());
+    for dir_entry in dir_entries.iter_mut() {
+        if dir_entry.is_dir {
+            names.push(dir_entry.name.clone().unwrap());
+        } else {
+            dir_entry.name = Some(dir_entry.path.file_name().unwrap().to_string_lossy().to_string());
+        }
     }
 
-    assert_eq!(names.len(), 3);
-    assert!(names.contains(&setup.config.stream.path));
+    // Root + streams + state + runtime
+    assert_eq!(names.len(), 4);
+    let mut has_stream = false;
+    for name in names {
+        if name.contains(&setup.config.stream.path) {
+            has_stream = true;
+        }
+    }
+    assert!(has_stream);
 }
 
-#[tokio::test]
+#[compio::test]
 async fn should_create_and_persist_stream() {
     let setup = TestSetup::init().await;
-    let mut system = System::new(
-        setup.config.clone(),
-        DataMaintenanceConfig::default(),
-        PersonalAccessTokenConfig::default(),
-    );
+    let server_config = ServerConfig {
+        system:  setup.config.clone(),
+        ..Default::default()
+    };
+    let shard = IggyShard::default_from_config(server_config);
     let stream_id = 1;
     let stream_name = "test";
     let session = Session::new(1, 1, SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1234));
-    system.init().await.unwrap();
+    let session = Rc::new(session);
+    shard.init().await.unwrap();
+    shard.add_active_session(session.clone());
 
-    system
-        .create_stream(&session, Some(stream_id), stream_name)
-        .await
+    shard
+        .create_stream_bypass_auth( Some(stream_id), stream_name)
         .unwrap();
+    let stream = shard.get_stream(&Identifier::numeric(stream_id).unwrap()).unwrap();
+    stream.persist().await.unwrap();
 
     assert_persisted_stream(&setup.config.get_streams_path(), stream_id).await;
 }
 
-#[tokio::test]
+#[compio::test]
 async fn should_create_and_persist_stream_with_automatically_generated_id() {
     let setup = TestSetup::init().await;
-    let mut system = System::new(
-        setup.config.clone(),
-        DataMaintenanceConfig::default(),
-        PersonalAccessTokenConfig::default(),
-    );
+    let server_config = ServerConfig {
+        system:  setup.config.clone(),
+        ..Default::default()
+    };
+    let shard = IggyShard::default_from_config(server_config);
     let stream_id = 1;
     let stream_name = "test";
+    shard.init().await.unwrap();
     let session = Session::new(1, 1, SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1234));
-    system.init().await.unwrap();
+    let session = Rc::new(session);
+    shard.add_active_session(session.clone());
 
-    system
-        .create_stream(&session, None, stream_name)
-        .await
+    let stream_id = shard
+        .create_stream_bypass_auth( None, stream_name)
         .unwrap();
+    let stream = shard.get_stream(&Identifier::numeric(stream_id).unwrap()).unwrap();
+    stream.persist().await;
 
     assert_persisted_stream(&setup.config.get_streams_path(), stream_id).await;
 }
 
-#[tokio::test]
+#[compio::test]
 async fn should_delete_persisted_stream() {
     let setup = TestSetup::init().await;
-    let mut system = System::new(
-        setup.config.clone(),
-        DataMaintenanceConfig::default(),
-        PersonalAccessTokenConfig::default(),
-    );
+    let server_config = ServerConfig {
+        system:  setup.config.clone(),
+        ..Default::default()
+    };
+    let shard = IggyShard::default_from_config(server_config);
     let stream_id = 1;
     let stream_name = "test";
     let session = Session::new(1, 1, SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1234));
-    system.init().await.unwrap();
-    system
+    let session = Rc::new(session);
+    shard.init().await.unwrap();
+    shard.add_active_session(session.clone());
+
+    shard
         .create_stream(&session, Some(stream_id), stream_name)
         .await
         .unwrap();
     assert_persisted_stream(&setup.config.get_streams_path(), stream_id).await;
-    let stream_path = system
+    let stream_path = shard
         .get_stream(&Identifier::numeric(stream_id).unwrap())
         .unwrap()
         .path
         .clone();
 
-    system
-        .delete_stream(&session, &Identifier::numeric(1).unwrap())
-        .await
+    let stream = shard
+        .delete_stream(&session, &Identifier::numeric(stream_id).unwrap())
         .unwrap();
-
+    for topic in stream.get_topics() {
+        let partitions = topic.get_partitions();
+        for partition in partitions {
+            let mut part = partition.write().await;
+            part.delete().await.unwrap();
+        }
+        topic.delete().await.unwrap();
+    }
+    stream.delete().await.unwrap();
     assert!(fs::metadata(stream_path).await.is_err());
 }
 
@@ -128,5 +155,3 @@ async fn assert_persisted_stream(streams_path: &str, stream_id: u32) {
     let stream_metadata = fs::metadata(stream_path).await.unwrap();
     assert!(stream_metadata.is_dir());
 }
-
-*/
diff --git a/core/server/src/io/fs_utils.rs b/core/server/src/io/fs_utils.rs
index 6bbc1b7f..60616287 100644
--- a/core/server/src/io/fs_utils.rs
+++ b/core/server/src/io/fs_utils.rs
@@ -24,18 +24,24 @@ use std::path::{Path, PathBuf};
 pub struct DirEntry {
     pub path: PathBuf,
     pub is_dir: bool,
+    pub name: Option<String>,
 }
 
 impl DirEntry {
-    fn file(path: PathBuf) -> Self {
+    fn file(path: PathBuf, name: Option<String>) -> Self {
         Self {
             path,
             is_dir: false,
+            name
         }
     }
 
-    fn dir(path: PathBuf) -> Self {
-        Self { path, is_dir: true }
+    fn dir(path: PathBuf, name: Option<String>) -> Self {
+        Self {
+            path,
+            is_dir: true,
+            name
+        }
     }
 }
 
@@ -59,7 +65,7 @@ pub async fn walk_dir(root: impl AsRef<Path>) -> io::Result<Vec<DirEntry>> {
     let mut stack = vec![root.to_path_buf()];
 
     while let Some(current_dir) = stack.pop() {
-        directories.push(DirEntry::dir(current_dir.clone()));
+        directories.push(DirEntry::dir(current_dir.clone(), current_dir.to_str().map(|s| s.to_string())));
 
         for entry in std::fs::read_dir(&current_dir)? {
             let entry = entry?;
@@ -69,13 +75,20 @@ pub async fn walk_dir(root: impl AsRef<Path>) -> io::Result<Vec<DirEntry>> {
             if metadata.is_dir() {
                 stack.push(entry_path);
             } else {
-                files.push(DirEntry::file(entry_path));
+                files.push(DirEntry::file(entry_path, entry.file_name().into_string().ok()));
             }
         }
     }
 
     directories.reverse();
     files.extend(directories);
+    for file in files.iter() {
+        if file.is_dir {
+            tracing::error!("Directory: {:?}", file.path);
+        } else {
+            tracing::error!("File: {:?} (name: {:?})", file.path, file.name);
+        }
+    }
     Ok(files)
 }
 
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index b8ba22b7..b884a92d 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -570,7 +570,8 @@ impl IggyShard {
     async fn handle_event(&self, event: Arc<ShardEvent>) -> Result<(), IggyError> {
         match &*event {
             ShardEvent::CreatedStream { stream_id, name } => {
-                self.create_stream_bypass_auth(*stream_id, name)
+                self.create_stream_bypass_auth(*stream_id, name)?;
+                Ok(())
             }
             ShardEvent::CreatedTopic {
                 stream_id,
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index b4b1b63a..3d5a7785 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -193,13 +193,13 @@ impl IggyShard {
         &self,
         stream_id: Option<u32>,
         name: &str,
-    ) -> Result<(), IggyError> {
+    ) -> Result<u32, IggyError> {
         let stream = self.create_stream_base(stream_id, name)?;
         let id = stream.stream_id;
         self.streams_ids.borrow_mut().insert(name.to_owned(), id);
         self.streams.borrow_mut().insert(id, stream);
         self.metrics.increment_streams(1);
-        Ok(())
+        Ok(id)
     }
 
     pub async fn create_stream(
@@ -208,7 +208,7 @@ impl IggyShard {
         stream_id: Option<u32>,
         name: &str,
     ) -> Result<Identifier, IggyError> {
-        self.ensure_authenticated(session)?;
+        self.ensure_authenticated(session);
         self.permissioner
             .borrow()
             .create_stream(session.get_user_id())?;
@@ -352,11 +352,15 @@ impl IggyShard {
 
     pub fn delete_stream(&self, session: &Session, id: &Identifier) -> Result<Stream, IggyError> {
         self.ensure_authenticated(session)?;
-        let stream = self.get_stream(id).with_error_context(|error| {
-            format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
-        })?;
-        let stream_id = stream.stream_id;
-        let stream_name = stream.name.clone();
+        let mut stream_id;
+        let mut stream_name;
+        {
+            let stream = self.get_stream(id).with_error_context(|error| {
+                format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {id}")
+            })?;
+            stream_id = stream.stream_id;
+            stream_name = stream.name.clone();
+        }
         self.permissioner
             .borrow()
             .delete_stream(session.get_user_id(), stream_id)
@@ -364,7 +368,7 @@ impl IggyShard {
                 format!(
                     "{COMPONENT} (error: {error}) - permission denied to delete stream for user {}, stream ID: {}",
                     session.get_user_id(),
-                    stream.stream_id,
+                    stream_id,
                 )
             })?;
         let stream = self.delete_stream_base(stream_id, stream_name)?;